<commit_before># -*- coding: utf-8 -*-
import logging
from collections import defaultdict
from datetime import datetime
from datetime import timedelta

from django.conf import settings
from django.http import JsonResponse
from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Match, Q, Range
from elasticsearch_dsl.connections import connections
from rest_framework.decorators import api_view

logger = logging.getLogger(__name__)

connections.create_connection(hosts=[settings.ES_URL], timeout=20)

# This maps the query parameters to factory methods for the queries
QUERIES = {
    'title': 'get_title',
    'speaker': 'get_speaker',
    'content': 'get_content',
    'entity': 'get_entity'
}

def make_search():
    return Search(index=settings.ES_CW_INDEX)

def execute_search(query, sorting='-date_issued'):
    """
    Runs a search with basic handling for sorting
    :param query: the query to execute
    :param sorting: the type of sorting, i.e. -dateIssued
    :return: results or False
    """
    search = make_search().sort(sorting)
    results = search.query(query).execute()
    if results.success():
        return results
    return False

def get_speaker(name):
    return Match(speakers={"query": name, "operator": "and"})

def get_title(title):
    return Match(title={"query": title, "type": "phrase"})

def get_content(content):
    return Q('match', content={'query': content, 'operator': 'and'})

def get_entity(entity):
    return Match(named_entities={"query": entity, "operator": "and"})

def get_date_range(start, end="now/d"):
    return Range(date_issued={"gte": start, "lte": end})

@api_view(['GET'])
def search_by_speaker(request, name):
    """
    Search for a speaker by name
    :param request:
    :param name: name of the congress person speaking
    :return: list sorted by date_issued
    """
    query = get_speaker(name)
    response = execute_search(query)
    if response.success():
        return JsonResponse(response.to_dict())

@api_view(['GET'])
def search_by_title(request, title):
    """
    Search by title of a document
    :param request:
    :param title: the title
    :return: list of results sorted by date_issued
    """
    query = get_title(title)
    response = execute_search(query)
    if response.success():
        return JsonResponse(response.to_dict())

@api_view(['GET'])
def search_by_entities(request):
    """
    Search by title of a document
    entities are specified in the query param 'entity'
    :param request:
    :return: list of results sorted by date_issued
    """
    terms = request.GET.getlist('entity')
    queries = [get_entity(e) for e in terms]
    logger.info("queries? %s " % queries)
    q = Q('bool', must=queries)
    response = execute_search(q)
    if response.success():
        return JsonResponse(response.to_dict())

@api_view(['GET'])
def search_by_params(request):
    """
    Search by arbitrary params which can be combined and can also be lists

    title - search on the title field
    speaker - individual speakers
    content - search for text matches in the content add &highlight=<number> to include contextual
              matches of the given fragment size in the result (default is 200)
    entity - named entities
    start_date / end_date in form 2017-01-04, inclusive. end_date defaults to today.

    Example:
        http://localhost:8000/cwapi/search/multi/?title=Budget&entity=social%20security&entity=senate&speaker=sanders
        http://localhost:8000/cwapi/search/multi/?speaker=schumer&content=trump&highlight=1000
        http://localhost:8000/cwapi/search/multi/?title=Budget&speaker=sanders&start_date=2016-01-11&end_date=2017-01-04

    :param request:
    :return:
    """
    logger.debug("search_by_params")
    params = request.GET
    search = make_search().sort('-date_issued')
    queries = []
    for f in QUERIES:
        if f in params:
            for r in params.getlist(f):
                queries.append(globals()[QUERIES[f]](r))
    if 'start_date' in params:
        if 'end_date' in params:
            queries.append(get_date_range(params.get('start_date'), params.get('end_date')))
        else:
            queries.append(get_date_range(params.get('start_date')))
    logger.info("queries? %s " % queries)
    q = Q('bool', must=queries)

    if 'highlight' in params and 'content' in params:
        frag_size = params.get('highlight')
        if not frag_size.isnumeric():
            frag_size = 200
        response = search.highlight('content', fragment_size=frag_size).query(q).execute()
    else:
        response = search.query(q).execute()
    if response.success():
        return JsonResponse(response.to_dict())
    return JsonResponse("Found nothing")

def count_term_in_range(term, start_date, end_date):
    print(start_date)
    print(end_date)
    query = get_content(term)
    search = make_search()

    # gte: Greater or Equal, lte: Better than 4G? JK Less than or Equal
    date_filter = {
        'range': {'date_issued': {'gte': start_date, 'lte': end_date}}
    }
    docs = search.query(query).filter(date_filter).execute()
    print(len(docs))
    grouped_by_day = defaultdict(int)
    for doc in docs:
        grouped_by_day[doc.date_issued] += 1
    return grouped_by_day

@api_view(['GET'])
def count_of_term_in_content(request, term):
    """
    Count the mention of a term in content

    time_period(optional, default to 'month') - period of time to go
                                                back from today
                                                month | week | day

    Example:
        http://127.0.0.1:8000/cwapi/count/obamacare/?time_period=week
        http://127.0.0.1:8000/cwapi/count/obamacare/

    :param request:
    :param term: term whose occurances we are counting
    :return: number of occurances of the term in content
    """
    days_ago = request.query_params.get('days_ago', 30)

    start_date = datetime.utcnow() - timedelta(days=days_ago)
    end_date = datetime.utcnow()
    start_date.replace(hour=0, minute=0, second=0, microsecond=0)
    end_date.replace(hour=0, minute=0, second=0, microsecond=0)

    # TODO: remove when we have more recent data
    start_date -= timedelta(days=150)
    end_date -= timedelta(days=150)

    current_period = count_term_in_range(term, start_date, end_date)
    prev_start_date = start_date - timedelta(days=days_ago)
    prev_end_date = end_date - timedelta(days=days_ago)
    prev_period = count_term_in_range(term, prev_start_date, prev_end_date)

    current_period_total = sum([v for k, v in current_period.items()])
    prev_period_total = sum([v for k, v in prev_period.items()])

    return JsonResponse(
        {
            'term': term,
            'current_period': {
                'daily_breakdown': current_period,
                'total_count': current_period_total,
            },
            'previous_period': {
                'daily_breakdown': prev_period,
                'total_count': prev_period_total,
            },
            'start_date': start_date,
            'end_date': end_date,
        }
    )
<commit_msg>[capitol-words] removed some printlning<commit_after># -*- coding: utf-8 -*-
import logging
from collections import defaultdict
from datetime import datetime
from datetime import timedelta

from django.conf import settings
from django.http import JsonResponse
from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Match, Q, Range
from elasticsearch_dsl.connections import connections
from rest_framework.decorators import api_view

logger = logging.getLogger(__name__)

connections.create_connection(hosts=[settings.ES_URL], timeout=20)

# This maps the query parameters to factory methods for the queries
QUERIES = {
    'title': 'get_title',
    'speaker': 'get_speaker',
    'content': 'get_content',
    'entity': 'get_entity'
}

def make_search():
    return Search(index=settings.ES_CW_INDEX)

def execute_search(query, sorting='-date_issued'):
    """
    Runs a search with basic handling for sorting
    :param query: the query to execute
    :param sorting: the type of sorting, i.e. -dateIssued
    :return: results or False
    """
    search = make_search().sort(sorting)
    results = search.query(query).execute()
    if results.success():
        return results
    return False

def get_speaker(name):
    return Match(speakers={"query": name, "operator": "and"})

def get_title(title):
    return Match(title={"query": title, "type": "phrase"})

def get_content(content):
    return Q('match', content={'query': content, 'operator': 'and'})

def get_entity(entity):
    return Match(named_entities={"query": entity, "operator": "and"})

def get_date_range(start, end="now/d"):
    return Range(date_issued={"gte": start, "lte": end})

@api_view(['GET'])
def search_by_speaker(request, name):
    """
    Search for a speaker by name
    :param request:
    :param name: name of the congress person speaking
    :return: list sorted by date_issued
    """
    query = get_speaker(name)
    response = execute_search(query)
    if response.success():
        return JsonResponse(response.to_dict())

@api_view(['GET'])
def search_by_title(request, title):
    """
    Search by title of a document
    :param request:
    :param title: the title
    :return: list of results sorted by date_issued
    """
    query = get_title(title)
    response = execute_search(query)
    if response.success():
        return JsonResponse(response.to_dict())

@api_view(['GET'])
def search_by_entities(request):
    """
    Search by title of a document
    entities are specified in the query param 'entity'
    :param request:
    :return: list of results sorted by date_issued
    """
    terms = request.GET.getlist('entity')
    queries = [get_entity(e) for e in terms]
    logger.info("queries? %s " % queries)
    q = Q('bool', must=queries)
    response = execute_search(q)
    if response.success():
        return JsonResponse(response.to_dict())

@api_view(['GET'])
def search_by_params(request):
    """
    Search by arbitrary params which can be combined and can also be lists

    title - search on the title field
    speaker - individual speakers
    content - search for text matches in the content add &highlight=<number> to include contextual
              matches of the given fragment size in the result (default is 200)
    entity - named entities
    start_date / end_date in form 2017-01-04, inclusive. end_date defaults to today.

    Example:
        http://localhost:8000/cwapi/search/multi/?title=Budget&entity=social%20security&entity=senate&speaker=sanders
        http://localhost:8000/cwapi/search/multi/?speaker=schumer&content=trump&highlight=1000
        http://localhost:8000/cwapi/search/multi/?title=Budget&speaker=sanders&start_date=2016-01-11&end_date=2017-01-04

    :param request:
    :return:
    """
    logger.debug("search_by_params")
    params = request.GET
    search = make_search().sort('-date_issued')
    queries = []
    for f in QUERIES:
        if f in params:
            for r in params.getlist(f):
                queries.append(globals()[QUERIES[f]](r))
    if 'start_date' in params:
        if 'end_date' in params:
            queries.append(get_date_range(params.get('start_date'), params.get('end_date')))
        else:
            queries.append(get_date_range(params.get('start_date')))
    logger.info("queries? %s " % queries)
    q = Q('bool', must=queries)

    if 'highlight' in params and 'content' in params:
        frag_size = params.get('highlight')
        if not frag_size.isnumeric():
            frag_size = 200
        response = search.highlight('content', fragment_size=frag_size).query(q).execute()
    else:
        response = search.query(q).execute()
    if response.success():
        return JsonResponse(response.to_dict())
    return JsonResponse("Found nothing")

def count_term_in_range(term, start_date, end_date):
    query = get_content(term)
    search = make_search()

    # gte: Greater or Equal, lte: Better than 4G? JK Less than or Equal
    date_filter = {
        'range': {'date_issued': {'gte': start_date, 'lte': end_date}}
    }
    docs = search.query(query).filter(date_filter).execute()
    grouped_by_day = defaultdict(int)
    for doc in docs:
        grouped_by_day[doc.date_issued] += 1
    return grouped_by_day

@api_view(['GET'])
def count_of_term_in_content(request, term):
    """
    Count the mention of a term in content

    time_period(optional, default to 'month') - period of time to go
                                                back from today
                                                month | week | day

    Example:
        http://127.0.0.1:8000/cwapi/count/obamacare/?time_period=week
        http://127.0.0.1:8000/cwapi/count/obamacare/

    :param request:
    :param term: term whose occurances we are counting
    :return: number of occurances of the term in content
    """
    days_ago = request.query_params.get('days_ago', 30)

    start_date = datetime.utcnow() - timedelta(days=days_ago)
    end_date = datetime.utcnow()
    start_date.replace(hour=0, minute=0, second=0, microsecond=0)
    end_date.replace(hour=0, minute=0, second=0, microsecond=0)

    # TODO: remove when we have more recent data
    start_date -= timedelta(days=150)
    end_date -= timedelta(days=150)

    current_period = count_term_in_range(term, start_date, end_date)
    prev_start_date = start_date - timedelta(days=days_ago)
    prev_end_date = end_date - timedelta(days=days_ago)
    prev_period = count_term_in_range(term, prev_start_date, prev_end_date)

    current_period_total = sum([v for k, v in current_period.items()])
    prev_period_total = sum([v for k, v in prev_period.items()])

    return JsonResponse(
        {
            'term': term,
            'current_period': {
                'daily_breakdown': current_period,
                'total_count': current_period_total,
            },
            'previous_period': {
                'daily_breakdown': prev_period,
                'total_count': prev_period_total,
            },
            'start_date': start_date,
            'end_date': end_date,
        }
    )
<|endoftext|>


<commit_before>import unittest

import deepchem as dc

class TestInit(unittest.TestCase):
  def test_version(self):
    assert dc.__version__
<commit_msg>made code formatting yapf compliant<commit_after>import unittest

import deepchem as dc

class TestInit(unittest.TestCase):

  def test_version(self):
    assert dc.__version__
<|endoftext|>


<commit_before>import wx
import os

class AddressBook(wx.Frame):
    def __init__(self):
        wx.Frame.__init__(self,None,-1,"Address Book",size=(520,700))
        self.panel=wx.Panel(self,-1)
        self.SetBackgroundColour("#95CAE4")

        self.title=wx.StaticText(self.panel,-1,"ADDRESS BOOK DATABASE",pos=(120,10))
        self.firstname=wx.StaticText(self.panel,-1,"FirstName",pos=(50,50))
        #self.firstname1=wx.TextCtrl(self.panel,-1,"",pos=(160,50),size=(100,20))
        self.secondname=wx.StaticText(self.panel,-1,"SecondName",pos=(35,80))
        #self.secondname1=wx.TextCtrl(self.panel,-1,"",pos=(160,80),size=(100,20))
        self.addressline1=wx.StaticText(self.panel,-1,"Address(Line1)",pos=(30,110))
        self.addressline2=wx.StaticText(self.panel,-1,"Address(Line2)",pos=(30,140))
        self.addressline3=wx.StaticText(self.panel,-1,"Address(Line3)",pos=(30,170))
        self.city=wx.StaticText(self.panel,-1,"City",pos=(85,200))
        self.state=wx.StaticText(self.panel,-1,"State or Province",pos=(20,230))
        self.zip=wx.StaticText(self.panel,-1,"Postal(zip)Code",pos=(25,260))
        self.homephe=wx.StaticText(self.panel,-1,"HomePhone",pos=(40,290))
        self.workphe=wx.StaticText(self.panel,-1,"WorkPhone",pos=(45,320))
        self.mobile=wx.StaticText(self.panel,-1,"Mobile",pos=(70,350))
        self.email=wx.StaticText(self.panel,-1,"Primary Email",pos=(35,380))
        self.email2=wx.StaticText(self.panel,-1,"Secondary Email",pos=(20,410))
        self.notes=wx.StaticText(self.panel,-1,"Notes",pos=(75,440))

        self.firstname1=wx.TextCtrl(self.panel,-1,"",pos=(130,50),size=(170,20))
        self.secondname1=wx.TextCtrl(self.panel,-1,"",pos=(130,80),size=(170,20))
        self.addressline11=wx.TextCtrl(self.panel,-1,"",pos=(130,110),size=(250,20))
        self.addressline21=wx.TextCtrl(self.panel,-1,"",pos=(130,140),size=(250,20))
        self.addressline31=wx.TextCtrl(self.panel,-1,"",pos=(130,170),size=(250,20))

        self.city1=wx.TextCtrl(self.panel,-1,"",pos=(130,200),size=(170,20))
        self.state1=wx.TextCtrl(self.panel,-1,"",pos=(130,230),size=(170,20))
        self.zip1=wx.TextCtrl(self.panel,-1,"",pos=(130,260),size=(170,20))
        self.homephe1=wx.TextCtrl(self.panel,-1,"",pos=(130,290),size=(170,20))
        self.workphe1=wx.TextCtrl(self.panel,-1,"",pos=(130,320),size=(170,20))
        self.mobile1=wx.TextCtrl(self.panel,-1,"",pos=(130,350),size=(170,20))
        self.email3=wx.TextCtrl(self.panel,-1,"",pos=(130,380),size=(170,20))
        self.email4=wx.TextCtrl(self.panel,-1,"",pos=(130,410),size=(170,20))
        self.notes1=wx.TextCtrl(self.panel,-1,"",pos=(130,440),size=(250,100),style=wx.TE_MULTILINE)
        self.btn=wx.Button(self.panel,201,"save",pos=(170,570),size=(70,30))
        self.btn1=wx.Button(self.panel,202,"clear all",pos=(80,570),size=(70,30))
        self.btn2=wx.Button(self.panel,203,"retrieve",pos=(260,570),size=(70,30))

        self.Bind(wx.EVT_BUTTON,self.retrive,id=203)
        self.Bind(wx.EVT_BUTTON,self.clear,id=202)
        self.Bind(wx.EVT_BUTTON,self.save,id=201)

        self.filename="AddressBookDataBase"
        if not os.path.exists(self.filename):
            os.makedirs('AddressBookDataBase')

    def save(self,event):
        self.firstname=self.firstname1.GetValue()
        self.secondname=self.secondname1.GetValue()
        self.addressline1=self.addressline11.GetValue()
        self.addressline2=self.addressline21.GetValue()
        self.addressline3=self.addressline31.GetValue()
        self.city=self.city1.GetValue()
        self.state=self.state1.GetValue()
        self.zip=self.zip1.GetValue()
        self.homephe=self.homephe1.GetValue()
        self.workphe=self.workphe1.GetValue()
        self.mobile=self.mobile1.GetValue()
        self.email=self.email3.GetValue()
        self.email2=self.email4.GetValue()
        self.notes=self.notes1.GetValue()
        if (len(self.firstname) and len(self.secondname) and len(self.addressline1) and len(self.addressline2) and len(self.addressline3) and len(self.city) and len(self.state) and len(self.zip) and len(self.homephe) and len(self.workphe) and len(self.mobile) and len(self.email) and len(self.email2) and len(self.notes))>0:

           # print self.firstname
            box=wx.TextEntryDialog(None,"Enter AddressBookName To Save","Title","default text")
            if box.ShowModal()==wx.ID_OK:
                self.addressbookname=box.GetValue()
                #self.dial = wx.MessageDialog(None, 'saved successfully', 'Info', wx.OK)
                #self.dial.ShowModal()
                if not os.path.exists('AddressBookDataBase/'+self.addressbookname+'.txt'):

                    self.filename = open('AddressBookDataBase/'+self.addressbookname+'.txt','w')
                    self.filename.write(self.firstname+'\n')
                    self.filename.write(self.secondname+'\n')
                    self.filename.write(self.addressline1+'\n')
                    self.filename.write(self.addressline2+'\n')
                    self.filename.write(self.addressline3+'\n')
                    self.filename.write(self.city+'\n')
                    self.filename.write(self.state+'\n')
                    self.filename.write(self.zip+'\n')
                    self.filename.write(self.homephe+'\n')
                    self.filename.write(self.workphe+'\n')
                    self.filename.write(self.mobile+'\n')
                    self.filename.write(self.email+'\n')
                    self.filename.write(self.email2+'\n')
                    self.filename.write(self.notes+'\n')
                    self.filename.close()
                    self.filename=open('AddressBookDataBase/filenames.txt','a')
                    self.filename.write(self.addressbookname+'\n')
                    self.filename.close()
                    self.dial = wx.MessageDialog(None, 'saved successfully', 'Info', wx.OK)
                    self.dial.ShowModal()

                else:
                    self.dial = wx.MessageDialog(None, 'AddressBookname already exists, enter another name to save', 'Info', wx.OK)
                    self.dial.ShowModal()

            else:
                self.dial = wx.MessageDialog(None, 'save cancelled', 'Info', wx.OK)
                self.dial.ShowModal()

        else:

            self.dial = wx.MessageDialog(None, 'please fill all fields', 'Info', wx.OK)
            self.dial.ShowModal()

    def clear(self,event):
        self.firstname1.Clear()
        self.secondname1.Clear()
        self.addressline11.Clear()
        self.addressline21.Clear()
        self.addressline31.Clear()
        self.city1.Clear()
        self.state1.Clear()
        self.zip1.Clear()
        self.homephe1.Clear()
        self.workphe1.Clear()
        self.mobile1.Clear()
        self.email3.Clear()
        self.email4.Clear()
        self.notes1.Clear()

    def retrive(self,event):

        self.fo=open('AddressBookDataBase/filenames.txt','r')
        f=self.fo.readlines()
        self.fo.close()

        self.box=wx.SingleChoiceDialog(None,"choose Address book name","filenames",f)
        if self.box.ShowModal()==wx.ID_OK:
            self.apples=self.box.GetStringSelection()
            self.app=self.apples[:-1]
            self.filenam = open('AddressBookDataBase/'+self.app+'.txt','r')
            line=self.filenam.readlines()
            line1=line[0]
            line2=line[1]
            line3=line[2]
            line4=line[3]
            line5=line[4]
            line6=line[5]
            line7=line[6]
            line8=line[7]
            line9=line[8]
            line10=line[9]
            line11=line[10]
            line12=line[11]
            line13=line[12]
            line14=line[13]
            self.firstname1.SetValue(line1[:-1])
            self.secondname1.SetValue(line2[:-1])
            self.addressline11.SetValue(line3[:-1])
            self.addressline21.SetValue(line4[:-1])
            self.addressline31.SetValue(line5[:-1])

            self.city1.SetValue(line6[:-1])
            self.state1.SetValue(line7[:-1])
            self.zip1.SetValue(line8[:-1])
            self.homephe1.SetValue(line9[:-1])
            self.workphe1.SetValue(line10[:-1])
            self.mobile1.SetValue(line11[:-1])
            self.email3.SetValue(line12[:-1])
            self.email4.SetValue(line13[:-1])
            self.notes1.SetValue(line14[:-1])
            self.btn4=wx.Button(self.panel,205,"update",pos=(350,570),size=(70,30))
            self.Bind(wx.EVT_BUTTON,self.update,id=205)
            self.note=wx.StaticText(self.panel,-1,"Note: update button is used to modify the existing address book",pos=(10,610))

    def update(self,event):

        self.firstname=self.firstname1.GetValue()
        print self.firstname
        self.secondname=self.secondname1.GetValue()
        self.addressline1=self.addressline11.GetValue()
        self.addressline2=self.addressline21.GetValue()
        self.addressline3=self.addressline31.GetValue()
        self.city=self.city1.GetValue()
        self.state=self.state1.GetValue()
        self.zip=self.zip1.GetValue()
        self.homephe=self.homephe1.GetValue()
        self.workphe=self.workphe1.GetValue()
        self.mobile=self.mobile1.GetValue()
        self.email=self.email3.GetValue()
        self.email2=self.email4.GetValue()
        self.notes=self.notes1.GetValue()

        self.updat = open('AddressBookDataBase/'+self.app+'.txt','w')
        self.updat.write(self.firstname+'\n')
        self.updat.write(self.secondname+'\n')
        self.updat.write(self.addressline1+'\n')
        self.updat.write(self.addressline2+'\n')
        self.updat.write(self.addressline3+'\n')
        self.updat.write(self.city+'\n')
        self.updat.write(self.state+'\n')
        self.updat.write(self.zip+'\n')
        self.updat.write(self.homephe+'\n')
        self.updat.write(self.workphe+'\n')
        self.updat.write(self.mobile+'\n')
        self.updat.write(self.email+'\n')
        self.updat.write(self.email2+'\n')
        self.updat.write(self.notes+'\n')
        self.updat.close()
        self.app=0
        self.btn4.Destroy()
        self.dial = wx.MessageDialog(None, 'updated', 'Info', wx.OK)
        self.dial.ShowModal()

app=wx.App()
frame=AddressBook()
frame.Centre()
frame.Show()
app.MainLoop()
<commit_msg>address book<commit_after>import wx
import os

class AddressBook(wx.Frame):
    def __init__(self):
        wx.Frame.__init__(self,None,-1,"Address Book",size=(520,700))
        self.panel=wx.Panel(self,-1)
        self.SetBackgroundColour("#95CAE4")

        self.title=wx.StaticText(self.panel,-1,"ADDRESS BOOK DATABASE",pos=(120,10))
        self.firstname=wx.StaticText(self.panel,-1,"FirstName",pos=(50,50))
        #self.firstname1=wx.TextCtrl(self.panel,-1,"",pos=(160,50),size=(100,20))
        self.secondname=wx.StaticText(self.panel,-1,"SecondName",pos=(35,80))
        #self.secondname1=wx.TextCtrl(self.panel,-1,"",pos=(160,80),size=(100,20))
        self.addressline1=wx.StaticText(self.panel,-1,"Address(Line1)",pos=(30,110))
        self.addressline2=wx.StaticText(self.panel,-1,"Address(Line2)",pos=(30,140))
        self.addressline3=wx.StaticText(self.panel,-1,"Address(Line3)",pos=(30,170))
        self.city=wx.StaticText(self.panel,-1,"City",pos=(85,200))
        self.state=wx.StaticText(self.panel,-1,"State or Province",pos=(20,230))
        self.zip=wx.StaticText(self.panel,-1,"Postal(zip)Code",pos=(25,260))
        self.homephe=wx.StaticText(self.panel,-1,"HomePhone",pos=(40,290))
        self.workphe=wx.StaticText(self.panel,-1,"WorkPhone",pos=(45,320))
        self.mobile=wx.StaticText(self.panel,-1,"Mobile",pos=(70,350))
        self.email=wx.StaticText(self.panel,-1,"Primary Email",pos=(35,380))
        self.email2=wx.StaticText(self.panel,-1,"Secondary Email",pos=(20,410))
        self.notes=wx.StaticText(self.panel,-1,"Notes",pos=(75,440))

        self.firstname1=wx.TextCtrl(self.panel,-1,"",pos=(130,50),size=(170,20))
        self.secondname1=wx.TextCtrl(self.panel,-1,"",pos=(130,80),size=(170,20))
        self.addressline11=wx.TextCtrl(self.panel,-1,"",pos=(130,110),size=(250,20))
        self.addressline21=wx.TextCtrl(self.panel,-1,"",pos=(130,140),size=(250,20))
        self.addressline31=wx.TextCtrl(self.panel,-1,"",pos=(130,170),size=(250,20))

        self.city1=wx.TextCtrl(self.panel,-1,"",pos=(130,200),size=(170,20))
        self.state1=wx.TextCtrl(self.panel,-1,"",pos=(130,230),size=(170,20))
        self.zip1=wx.TextCtrl(self.panel,-1,"",pos=(130,260),size=(170,20))
        self.homephe1=wx.TextCtrl(self.panel,-1,"",pos=(130,290),size=(170,20))
        self.workphe1=wx.TextCtrl(self.panel,-1,"",pos=(130,320),size=(170,20))
        self.mobile1=wx.TextCtrl(self.panel,-1,"",pos=(130,350),size=(170,20))
        self.email3=wx.TextCtrl(self.panel,-1,"",pos=(130,380),size=(170,20))
        self.email4=wx.TextCtrl(self.panel,-1,"",pos=(130,410),size=(170,20))
        self.notes1=wx.TextCtrl(self.panel,-1,"",pos=(130,440),size=(250,100),style=wx.TE_MULTILINE)
        self.btn=wx.Button(self.panel,201,"save",pos=(170,570),size=(70,30))
        self.btn1=wx.Button(self.panel,202,"clear all",pos=(80,570),size=(70,30))
        self.btn2=wx.Button(self.panel,203,"retrieve",pos=(260,570),size=(70,30))

        self.Bind(wx.EVT_BUTTON,self.retrive,id=203)
        self.Bind(wx.EVT_BUTTON,self.clear,id=202)
        self.Bind(wx.EVT_BUTTON,self.save,id=201)

        self.filename="AddressBookDataBase"
        if not os.path.exists(self.filename):
            os.makedirs('AddressBookDataBase')

    def save(self,event):
        self.firstname=self.firstname1.GetValue()
        self.secondname=self.secondname1.GetValue()
        self.addressline1=self.addressline11.GetValue()
        self.addressline2=self.addressline21.GetValue()
        self.addressline3=self.addressline31.GetValue()
        self.city=self.city1.GetValue()
        self.state=self.state1.GetValue()
        self.zip=self.zip1.GetValue()
        self.homephe=self.homephe1.GetValue()
        self.workphe=self.workphe1.GetValue()
        self.mobile=self.mobile1.GetValue()
        self.email=self.email3.GetValue()
        self.email2=self.email4.GetValue()
        self.notes=self.notes1.GetValue()
        if (len(self.firstname) and len(self.secondname) and len(self.addressline1) and len(self.addressline2) and len(self.addressline3) and len(self.city) and len(self.state) and len(self.zip) and len(self.homephe) and len(self.workphe) and len(self.mobile) and len(self.email) and len(self.email2) and len(self.notes))>0:

           # print self.firstname
            box=wx.TextEntryDialog(None,"Enter AddressBookName To Save","Title","default text")
            if box.ShowModal()==wx.ID_OK:
                self.addressbookname=box.GetValue()
                #self.dial = wx.MessageDialog(None, 'saved successfully', 'Info', wx.OK)
                #self.dial.ShowModal()
                if not os.path.exists('AddressBookDataBase/'+self.addressbookname+'.txt'):

                    self.filename = open('AddressBookDataBase/'+self.addressbookname+'.txt','w')
                    self.filename.write(self.firstname+'\n')
                    self.filename.write(self.secondname+'\n')
                    self.filename.write(self.addressline1+'\n')
                    self.filename.write(self.addressline2+'\n')
                    self.filename.write(self.addressline3+'\n')
                    self.filename.write(self.city+'\n')
                    self.filename.write(self.state+'\n')
                    self.filename.write(self.zip+'\n')
                    self.filename.write(self.homephe+'\n')
                    self.filename.write(self.workphe+'\n')
                    self.filename.write(self.mobile+'\n')
                    self.filename.write(self.email+'\n')
                    self.filename.write(self.email2+'\n')
                    self.filename.write(self.notes+'\n')
                    self.filename.close()
                    self.filename=open('AddressBookDataBase/filenames.txt','a')
                    self.filename.write(self.addressbookname+'\n')
                    self.filename.close()
                    self.dial = wx.MessageDialog(None, 'saved successfully', 'Info', wx.OK)
                    self.dial.ShowModal()

                else:
                    self.dial = wx.MessageDialog(None, 'AddressBookname already exists, enter another name to save', 'Info', wx.OK)
                    self.dial.ShowModal()

            else:
                self.dial = wx.MessageDialog(None, 'save cancelled', 'Info', wx.OK)
                self.dial.ShowModal()

        else:

            self.dial = wx.MessageDialog(None, 'please fill all fields', 'Info', wx.OK)
            self.dial.ShowModal()

    def clear(self,event):
        self.firstname1.Clear()
        self.secondname1.Clear()
        self.addressline11.Clear()
        self.addressline21.Clear()
        self.addressline31.Clear()
        self.city1.Clear()
        self.state1.Clear()
        self.zip1.Clear()
        self.homephe1.Clear()
        self.workphe1.Clear()
        self.mobile1.Clear()
        self.email3.Clear()
        self.email4.Clear()
        self.notes1.Clear()

    def retrive(self,event):

        self.fo=open('AddressBookDataBase/filenames.txt','r')
        f=self.fo.readlines()
        self.fo.close()

        self.box=wx.SingleChoiceDialog(None,"choose Address book name","filenames",f)
        if self.box.ShowModal()==wx.ID_OK:
            self.apples=self.box.GetStringSelection()
            self.app=self.apples[:-1]
            self.filenam = open('AddressBookDataBase/'+self.app+'.txt','r')
            line=self.filenam.readlines()
            line1=line[0]
            line2=line[1]
            line3=line[2]
            line4=line[3]
            line5=line[4]
            line6=line[5]
            line7=line[6]
            line8=line[7]
            line9=line[8]
            line10=line[9]
            line11=line[10]
            line12=line[11]
            line13=line[12]
            line14=line[13]
            self.firstname1.SetValue(line1[:-1])
            self.secondname1.SetValue(line2[:-1])
            self.addressline11.SetValue(line3[:-1])
            self.addressline21.SetValue(line4[:-1])
            self.addressline31.SetValue(line5[:-1])

            self.city1.SetValue(line6[:-1])
            self.state1.SetValue(line7[:-1])
            self.zip1.SetValue(line8[:-1])
            self.homephe1.SetValue(line9[:-1])
            self.workphe1.SetValue(line10[:-1])
            self.mobile1.SetValue(line11[:-1])
            self.email3.SetValue(line12[:-1])
            self.email4.SetValue(line13[:-1])
            self.notes1.SetValue(line14[:-1])
            self.btn4=wx.Button(self.panel,205,"update",pos=(350,570),size=(70,30))
            self.Bind(wx.EVT_BUTTON,self.update,id=205)
            self.note=wx.StaticText(self.panel,-1,"Note: After modify the existing address book HIT UPDATE button",pos=(10,610))

    def update(self,event):

        self.firstname=self.firstname1.GetValue()
        print self.firstname
        self.secondname=self.secondname1.GetValue()
        self.addressline1=self.addressline11.GetValue()
        self.addressline2=self.addressline21.GetValue()
        self.addressline3=self.addressline31.GetValue()
        self.city=self.city1.GetValue()
        self.state=self.state1.GetValue()
        self.zip=self.zip1.GetValue()
        self.homephe=self.homephe1.GetValue()
        self.workphe=self.workphe1.GetValue()
        self.mobile=self.mobile1.GetValue()
        self.email=self.email3.GetValue()
        self.email2=self.email4.GetValue()
        self.notes=self.notes1.GetValue()

        self.updat = open('AddressBookDataBase/'+self.app+'.txt','w')
        self.updat.write(self.firstname+'\n')
        self.updat.write(self.secondname+'\n')
        self.updat.write(self.addressline1+'\n')
        self.updat.write(self.addressline2+'\n')
        self.updat.write(self.addressline3+'\n')
        self.updat.write(self.city+'\n')
        self.updat.write(self.state+'\n')
        self.updat.write(self.zip+'\n')
        self.updat.write(self.homephe+'\n')
        self.updat.write(self.workphe+'\n')
        self.updat.write(self.mobile+'\n')
        self.updat.write(self.email+'\n')
        self.updat.write(self.email2+'\n')
        self.updat.write(self.notes+'\n')
        self.updat.close()
        self.app=0
        self.btn4.Destroy()
        self.dial = wx.MessageDialog(None, 'updated', 'Info', wx.OK)
        self.dial.ShowModal()

app=wx.App()
frame=AddressBook()
frame.Centre()
frame.Show()
app.MainLoop()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Created 03/20/15 by Jason Albert
# Program to deconstruct a VPD image into template files

############################################################
# Imports - Imports - Imports - Imports - Imports - Imports
############################################################
import os
# Get the path the script resides in
scriptPath = os.path.dirname(os.path.realpath(__file__))
import sys
sys.path.insert(0,scriptPath + "/pymod");
import cmdline
import out
import xml.etree.ElementTree as ET
import struct
import re

############################################################
# Classes - Classes - Classes - Classes - Classes - Classes
############################################################
class RecordInfo:
    """Stores the info about each vpd record"""
    def __init__(self):
        # The name of the record where the toc entry is located
        self.tocName = None
        # The location of the Record Offset
        self.recordOffset = None
        # The location of the Record Length
        self.recordLength = None
        # The location of the ECC Offset
        self.eccOffset = None
        # The location of the ECC Length
        self.eccLength = None

############################################################
# Function - Functions - Functions - Functions - Functions
############################################################
def help():
    out.msg("reverseVpd.py -v image.vpd -o outputpath -d")
    out.msg("Required Args")
    out.setIndent(2)
    out.msg("-v|--vpdfile           The valid vpd formatted input file")
    out.msg("-o|--outpath           The output path for the files created by the tool")
    out.setIndent(0)
    out.msg("Optional Args")
    out.setIndent(2)
    out.msg("-d|--debug             Enables debug printing")
    out.msg("-h|--help              This help text")
    out.msg("-r|--create-records    Create tvpd files for each record in the vpd")
    out.setIndent(0)

# Function to write out the resultant tvpd xml file
def writeTvpd(manifest, outputFile):
    tree = ET.ElementTree(manifest)
    tree.write(outputFile, encoding="utf-8", xml_declaration=True)
    return None

############################################################
# Main - Main - Main - Main - Main - Main - Main - Main
############################################################
rc = 0

################################################
# Command line options
clErrors = 0

# Help text
if (cmdline.parseOption("-h","--help")):
    help()
    exit(0)

# Get the manifest file and get this party started
clVpdFile = cmdline.parseOptionWithArg("-v", "--vpdfile")
if (clVpdFile == None):
    out.error("The -v arg is required!")
    clErrors+=1

# Look for output path
clOutputPath = cmdline.parseOptionWithArg("-o", "--outpath")
if (clOutputPath == None):
    out.error("The -o arg is required")
    clErrors+=1
else:
    # Make sure the path exists, we aren't going to create it
    if (os.path.exists(clOutputPath) != True):
        out.error("The given output path %s does not exist!" % clOutputPath)
        out.error("Please create the output directory and run again")
        clErrors+=1

# Error check the command line
if (clErrors):
    out.error("Missing/incorrect required cmdline args!  Please review the output above to determine which ones!")
    exit(clErrors)

# Debug printing
clDebug = cmdline.parseOption("-d", "--debug")

# Create separate binary files for each record
clBinaryRecords = cmdline.parseOption("-r", "--create-records")

# All cmdline args should be processed, so if any left throw an error
if (len(sys.argv) != 1):
    out.error("Extra cmdline args detected - %s" % (sys.argv[1:])) # [1:] don't inclue script name in the list
    exit(len(sys.argv))

################################################
# Read in the VPD file and break it apart
out.setIndent(0)
out.msg("==== Stage 1: Parsing the VPD file")
out.setIndent(2)

# Create our output name from the input name
vpdName = os.path.splitext(os.path.basename(clVpdFile))[0]

# Open the vpdfile
vpdContents = open(clVpdFile, mode='rb').read()

# Jump right to where the VTOC should be and make sure it says VTOC
offset = 61
if (vpdContents[offset:(offset+4)].decode() != "VTOC"):
    out.error("Did not find VTOC at the expected offset!")
    exit(1)
offset+=4

# Skip the PT keyword and read the 1 byte length to loop over the VTOC contents and create our record list
offset+=1 # PT skip
tocLength = vpdContents[offset]
offset+=1

# Keep a dictionary of the record names we come across
recordNames = dict()

# Loop through the toc and read out the record locations
tocOffset = 0
while (tocOffset < tocLength):
    # Get the record name
    recordName = vpdContents[(tocOffset + offset):(tocOffset + offset + 4)].decode()
    # Create the entry with the name
    recordNames[recordName] = RecordInfo()
    tocOffset+=4
    # Skip the record type
    tocOffset+=2
    # recordOffset
    recordNames[recordName].recordOffset = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])
    tocOffset+=2
    # recordLength
    recordNames[recordName].recordLength = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])
    tocOffset+=2
    # eccOffset
    recordNames[recordName].eccOffset = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])
    tocOffset+=2
    # eccLength
    recordNames[recordName].eccLength = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])
    tocOffset+=2

print("yeah!")

exit(0)

################################################
# Verify the tvpd XML
# read thru the complete tvpd and verify/check actual tag contents
out.setIndent(0)
out.msg("==== Stage 2: Verifying tvpd syntax")
out.setIndent(2)
errorsFound = 0

# We now have a correct tvpd, use it to create a binary VPD image
out.setIndent(0)
out.msg("==== Stage 3: Writing the tvpd output file")
out.setIndent(2)
# Create our output file names
tvpdFileName = clOutputPath + "/" + vpdName + ".tvpd"

# This is our easy one, write the XML back out
# Write out the full template vpd representing the data contained in our image
writeTvpd(manifest, tvpdFileName)
out.msg("Wrote tvpd file: %s" % tvpdFileName)

<commit_msg>A large portion of the tool works, just need to fix writing out hex data<commit_after>#!/usr/bin/env python
# Created 03/20/15 by Jason Albert
# Program to deconstruct a VPD image into template files

############################################################
# Imports - Imports - Imports - Imports - Imports - Imports
############################################################
import os
# Get the path the script resides in
scriptPath = os.path.dirname(os.path.realpath(__file__))
import sys
sys.path.insert(0,scriptPath + "/pymod");
import cmdline
import out
import xml.etree.ElementTree as ET
import struct
import re
import binascii

############################################################
# Classes - Classes - Classes - Classes - Classes - Classes
############################################################
class RecordInfo:
    """Stores the info about each vpd record"""
    def __init__(self):
        # The name of the record where the toc entry is located
        self.tocName = None
        # The location of the Record Offset
        self.recordOffset = None
        # The location of the Record Length
        self.recordLength = None
        # The location of the ECC Offset
        self.eccOffset = None
        # The location of the ECC Length
        self.eccLength = None

############################################################
# Function - Functions - Functions - Functions - Functions
############################################################
def help():
    out.msg("reverseVpd.py -v image.vpd -o outputpath -d")
    out.msg("Required Args")
    out.setIndent(2)
    out.msg("-v|--vpdfile           The valid vpd formatted input file")
    out.msg("-o|--outpath           The output path for the files created by the tool")
    out.setIndent(0)
    out.msg("Optional Args")
    out.setIndent(2)
    out.msg("-d|--debug             Enables debug printing")
    out.msg("-h|--help              This help text")
    out.msg("-r|--create-records    Create tvpd files for each record in the vpd")
    out.setIndent(0)

# Function to write out the resultant tvpd xml file
def writeTvpd(manifest, outputFile):
    tree = ET.ElementTree(manifest)
    tree.write(outputFile, encoding="utf-8", xml_declaration=True)
    return None

############################################################
# Main - Main - Main - Main - Main - Main - Main - Main
############################################################
rc = 0

################################################
# Command line options
clErrors = 0

# Help text
if (cmdline.parseOption("-h","--help")):
    help()
    exit(0)

# Get the manifest file and get this party started
clVpdFile = cmdline.parseOptionWithArg("-v", "--vpdfile")
if (clVpdFile == None):
    out.error("The -v arg is required!")
    clErrors+=1

# Look for output path
clOutputPath = cmdline.parseOptionWithArg("-o", "--outpath")
if (clOutputPath == None):
    out.error("The -o arg is required")
    clErrors+=1
else:
    # Make sure the path exists, we aren't going to create it
    if (os.path.exists(clOutputPath) != True):
        out.error("The given output path %s does not exist!" % clOutputPath)
        out.error("Please create the output directory and run again")
        clErrors+=1

# Error check the command line
if (clErrors):
    out.error("Missing/incorrect required cmdline args!  Please review the output above to determine which ones!")
    exit(clErrors)

# Debug printing
clDebug = cmdline.parseOption("-d", "--debug")

# Create separate binary files for each record
clBinaryRecords = cmdline.parseOption("-r", "--create-records")

# All cmdline args should be processed, so if any left throw an error
if (len(sys.argv) != 1):
    out.error("Extra cmdline args detected - %s" % (sys.argv[1:])) # [1:] don't inclue script name in the list
    exit(len(sys.argv))

################################################
# Read in the VPD file and break it apart
out.setIndent(0)
out.msg("==== Stage 1: Parsing the VPD file")
out.setIndent(2)

# Create our output name from the input name
vpdName = os.path.splitext(os.path.basename(clVpdFile))[0]

# Open the vpdfile
vpdContents = open(clVpdFile, mode='rb').read()

# Jump right to where the VTOC should be and make sure it says VTOC
offset = 61
if (vpdContents[offset:(offset+4)].decode() != "VTOC"):
    out.error("Did not find VTOC at the expected offset!")
    exit(1)
offset+=4

# Skip the PT keyword and read the 1 byte length to loop over the VTOC contents and create our record list
offset+=2 # PT skip
tocLength = vpdContents[offset]
print("tocLength: %d" % tocLength)
offset+=1

# Keep a dictionary of the record names we come across
recordNames = dict()

# Loop through the toc and read out the record locations
tocOffset = 0
while (tocOffset < tocLength):
    # Get the record name
    recordName = vpdContents[(tocOffset + offset):(tocOffset + offset + 4)].decode()
    # Create the entry with the name
    recordNames[recordName] = RecordInfo()
    tocOffset+=4
    # Skip the record type
    tocOffset+=2
    # recordOffset
    recordNames[recordName].recordOffset = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])[0]
    print("recordOffset: ", recordNames[recordName].recordOffset)
    tocOffset+=2
    # recordLength
    recordNames[recordName].recordLength = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])[0]
    tocOffset+=2
    # eccOffset
    recordNames[recordName].eccOffset = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])[0]
    tocOffset+=2
    # eccLength
    recordNames[recordName].eccLength = struct.unpack('<H', vpdContents[(tocOffset + offset):(tocOffset + offset + 2)])[0]
    tocOffset+=2

# We have all the record offsets in memory along with the entire VPD image.
# Go onto our next step and create XML in memory

################################################
# Create tvpd XML
out.setIndent(0)
out.msg("==== Stage 2: Creating tvpd XML")
out.setIndent(2)

# Create our top level level XML
vpd = ET.Element("vpd")

# Loop thru our records and create our record/keyword entries
for recordName in recordNames:
    # Create our record
    record=ET.SubElement(vpd, "record", {'name':recordName})

    # Start walking thru our record reading keywords out
    # As we get to each keyword, we'll create the keyword tag and it's sub tags
    recordOffset = recordNames[recordName].recordOffset

    # Skip the LR tag
    recordOffset+=1

    # Get the length
    recordLength = struct.unpack('<H', vpdContents[recordOffset:(recordOffset + 2)])[0]
    recordOffset+=2

    # Now loop and read until we get until the end of the record
    recordEnd = recordOffset + recordLength
    print("recordOffset: %d, recordLength: %d" % (recordOffset, recordLength))
    while (recordOffset < recordEnd):
        # Read the keyword
        keywordName = vpdContents[recordOffset:(recordOffset + 2)].decode()
        print("keyword: %s" % keywordName)
        recordOffset+=2

        # Determine if length is 1 or 2 bytes
        if (keywordName[0] == "#"):
            keywordLength = struct.unpack('<H', vpdContents[recordOffset:(recordOffset + 2)])[0]
            recordOffset+=2
        else:
            keywordLength = vpdContents[recordOffset]
            recordOffset+=1

        # Get the keyword data out
        keywordData = vpdContents[recordOffset:(recordOffset + keywordLength)]
        recordOffset+=keywordLength

        # If the keyword is PF, we are at the end at skip it
        if (keywordName == "PF"):
            continue

        # Create our keyword tag and subtags
        keyword=ET.SubElement(record, "keyword", {"name":keywordName})
        ET.SubElement(keyword, "kwdesc").text = "The " + keywordName + " keyword"
        allAscii = keywordData.isalpha()
        if (allAscii):
            ET.SubElement(keyword, "kwformat").text = "ascii"
        else:
            ET.SubElement(keyword, "kwformat").text = "hex"
        ET.SubElement(keyword, "kwlen").text = str(keywordLength)
        if (allAscii):
            ET.SubElement(keyword, "kwdata").text = keywordData.decode()
        else:
            ET.SubElement(keyword, "kwdata").text = binascii.hexlify(keywordData)

    # We should be done with all the keywords, which means it's pointing to the SR tag
    if (vpdContents[recordOffset] != 0x78):
        out.error("Small resource tag not found!")
        exit(1)

# We now have a correct tvpd, use it to create a binary VPD image
out.setIndent(0)
out.msg("==== Stage 3: Writing the tvpd output file")
out.setIndent(2)
# Create our output file names
tvpdFileName = clOutputPath + "/" + vpdName + ".tvpd"

# This is our easy one, write the XML back out
# Write out the full template vpd representing the data contained in our image
writeTvpd(vpd, tvpdFileName)
out.msg("Wrote tvpd file: %s" % tvpdFileName)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.7 on 2017-12-04 20:12
from __future__ import unicode_literals

import iso639

from dash.utils import chunks
from django.db import migrations, transaction
from django.db.models import Q

migration_lang_cache = {}

# As iso639-2 languages can be broad, not all iso639-2 languages have direct translations to iso639-3. This table
# maps country and iso639-2 codes to a specific iso639-3 language code. It isn't inclusive but covers the cases
# we know about at the time of our 639-2 -> 639-3 migration.
MIGRATION_OVERRIDES = {
    'Africa/Lagos:cpe': 'pcm',
    'Africa/Monrovia:cpe': 'lir',
    'America/Managua:cpe': 'bzk',
    '*:mkh': 'khm',
    '*:cpe': 'pcm',
    '*:art': 'epo',
    '*:cpf': 'gcr',
    '*:phi': 'fil',
    '*:smi': 'smj',
    '*:afa': 'ara',
    '*:aus': 'rop',
    '*:cpp': 'kea',
    '*:him': 'xnr',
    '*:kar': 'blk',
    '*:esp': 'spa',
}

def iso6392_to_iso6393(iso_code, timezone=None):
    """
    Given an iso639-2 code and an optional timezone name, returns the appropriate 639-3 code to use.
    """
    if iso_code is None:
        return None

    iso_code = iso_code.lower().strip()

    if iso_code == '':
        raise ValueError('iso_code must not be empty')

    cache_key = '{}:{}'.format('XX' if timezone is None else timezone, iso_code)

    if cache_key not in migration_lang_cache:

        # build our key
        override_key = '%s:%s' % (timezone, iso_code) if timezone else 'XX:%s' % iso_code
        override = MIGRATION_OVERRIDES.get(override_key)

        if not override and timezone:
            override_key = 'XX:%s' % iso_code
            override = MIGRATION_OVERRIDES.get(override_key)

        if override:
            return override

        else:
            # first try looking up by part 2 bibliographic (which is what we use when available)
            try:
                lang = iso639.languages.get(part2b=iso_code)
            except KeyError:
                lang = None

            # if not found, back down to typographical
            if lang is None:
                try:
                    lang = iso639.languages.get(part2t=iso_code)
                except KeyError:
                    pass
            # if not found, maybe it's already a iso639-3 code
            if lang is None:
                try:
                    lang = iso639.languages.get(part3=iso_code)
                except KeyError:
                    pass

            if lang and lang.part3:
                migration_lang_cache[cache_key] = lang.part3
                return lang.part3
    else:
        return migration_lang_cache[cache_key]

    raise ValueError("unable to determine iso639-3 code: %s (%s)" % (iso_code, timezone))

def migrate_language(Contact, FAQ):
    contacts = Contact.objects.exclude(Q(language=None) | Q(language="") | Q(language="eng"))
    total_contacts = contacts.count()

    if total_contacts:
        print("Found %d contacts to migrate languages for...")
        num_updated = 0

        for id_batch in chunks(contacts.only('id'), 1000):
            with transaction.atomic():
                contact_batch = Contact.objects.filter(id__in=id_batch).select_related('org')

                for contact in contact_batch:
                    new_language = iso6392_to_iso6393(contact.language, contact.org.timezone)

                    if new_language != contact.language:
                        contact.language = new_language
                        contact.save(update_fields=('language',))

                    num_updated += 1

            print(" > Updated %d of %d contacts" % (num_updated, total_contacts))

    faqs = list(FAQ.objects.exclude(Q(language=None) | Q(language="") | Q(language="eng")).select_related('org'))

    if len(faqs):
        print("Found %d FAQs to migrate languages for...")

        for faq in faqs:
            new_language = iso6392_to_iso6393(faq.language, faq.org.timezone)

            if new_language != faq.language:
                faq.language = new_language
                faq.save(update_fields=('language',))

def apply_manual():
    from casepro.contacts.models import Contact
    from casepro.msgs.models import FAQ
    migrate_language(Contact, FAQ)

def apply_as_migration(apps, schema_editor):
    Contact = apps.get_model('contacts', 'Contact')
    FAQ = apps.get_model('msgs', 'FAQ')
    migrate_language(Contact, FAQ)

class Migration(migrations.Migration):

    dependencies = [
        ('cases', '0045_auto_20161014_1341'),
        ('msgs', '0053_faq'),
    ]

    operations = [
        migrations.RunPython(apply_as_migration)
    ]
<commit_msg>Fixes to language migration<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.7 on 2017-12-04 20:12
from __future__ import unicode_literals

import iso639

from dash.utils import chunks
from django.db import migrations, transaction
from django.db.models import Q

migration_lang_cache = {}

# As iso639-2 languages can be broad, not all iso639-2 languages have direct translations to iso639-3. This table
# maps country and iso639-2 codes to a specific iso639-3 language code. It isn't inclusive but covers the cases
# we know about at the time of our 639-2 -> 639-3 migration.
MIGRATION_OVERRIDES = {
    'Africa/Lagos:cpe': 'pcm',
    'Africa/Monrovia:cpe': 'lir',
    'America/Managua:cpe': 'bzk',
    '*:mkh': 'khm',
    '*:cpe': 'pcm',
    '*:art': 'epo',
    '*:cpf': 'gcr',
    '*:phi': 'fil',
    '*:smi': 'smj',
    '*:afa': 'ara',
    '*:aus': 'rop',
    '*:cpp': 'kea',
    '*:him': 'xnr',
    '*:kar': 'blk',
    '*:esp': 'spa',
}

def iso6392_to_iso6393(iso_code, timezone=None):
    """
    Given an iso639-2 code and an optional timezone name, returns the appropriate 639-3 code to use.
    """
    if iso_code is None:
        return None

    iso_code = iso_code.lower().strip()

    if iso_code == '':
        raise ValueError('iso_code must not be empty')

    cache_key = '{}:{}'.format('*' if timezone is None else timezone, iso_code)

    if cache_key not in migration_lang_cache:

        # build our key
        override_key = '%s:%s' % (timezone, iso_code) if timezone else '*:%s' % iso_code
        override = MIGRATION_OVERRIDES.get(override_key)

        if not override and timezone:
            override_key = '*:%s' % iso_code
            override = MIGRATION_OVERRIDES.get(override_key)

        if override:
            return override

        else:
            # first try looking up by part 2 bibliographic (which is what we use when available)
            try:
                lang = iso639.languages.get(part2b=iso_code)
            except KeyError:
                lang = None

            # if not found, back down to typographical
            if lang is None:
                try:
                    lang = iso639.languages.get(part2t=iso_code)
                except KeyError:
                    pass
            # if not found, maybe it's already a iso639-3 code
            if lang is None:
                try:
                    lang = iso639.languages.get(part3=iso_code)
                except KeyError:
                    pass

            if lang and lang.part3:
                migration_lang_cache[cache_key] = lang.part3
                return lang.part3
    else:
        return migration_lang_cache[cache_key]

    raise ValueError("unable to determine iso639-3 code: %s (%s)" % (iso_code, timezone))

def migrate_language(Contact, FAQ):
    contacts = Contact.objects.exclude(Q(language=None) | Q(language="") | Q(language="eng"))
    total_contacts = contacts.count()

    if total_contacts:
        print("Found %d contacts to migrate languages for..." % total_contacts)
        num_updated = 0

        for batch in chunks(contacts.only('id'), 1000):
            with transaction.atomic():
                contact_batch = Contact.objects.filter(id__in=[c.id for c in batch]).select_related('org')

                for contact in contact_batch:
                    new_language = iso6392_to_iso6393(contact.language, contact.org.timezone)

                    if new_language != contact.language:
                        contact.language = new_language
                        contact.save(update_fields=('language',))

                    num_updated += 1

            print(" > Updated %d of %d contacts" % (num_updated, total_contacts))

    faqs = list(FAQ.objects.exclude(Q(language=None) | Q(language="") | Q(language="eng")).select_related('org'))

    if len(faqs):
        print("Found %d FAQs to migrate languages for..." % len(faqs))

        for faq in faqs:
            new_language = iso6392_to_iso6393(faq.language, faq.org.timezone)

            if new_language != faq.language:
                faq.language = new_language
                faq.save(update_fields=('language',))

def apply_manual():
    from casepro.contacts.models import Contact
    from casepro.msgs.models import FAQ
    migrate_language(Contact, FAQ)

def apply_as_migration(apps, schema_editor):
    Contact = apps.get_model('contacts', 'Contact')
    FAQ = apps.get_model('msgs', 'FAQ')
    migrate_language(Contact, FAQ)

class Migration(migrations.Migration):

    dependencies = [
        ('cases', '0045_auto_20161014_1341'),
        ('msgs', '0053_faq'),
    ]

    operations = [
        migrations.RunPython(apply_as_migration)
    ]
<|endoftext|>


<commit_before>"""
Process an input dataset into a format suitable for machine learning.
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
import multiprocessing as mp
import os
import cPickle as pickle
import gzip
import pandas as pd
import numpy as np
import csv
from functools import partial
from rdkit import Chem
from vs_utils.features.fingerprints import CircularFingerprint
from vs_utils.features.basic import SimpleDescriptors
from deepchem.utils.dataset import save_to_disk
from deepchem.utils.dataset import load_from_disk

def _process_field(val):
  """Parse data in a field."""
  if isinstance(val, float) or isinstance(val, np.ndarray):
    return val
  elif isinstance(val, list):
    return [process_field(elt) for elt in val]
  elif isinstance(val, str):
    try:
      return float(val)
    except ValueError:
      return val
  else:
    raise ValueError("Field of unrecognized type: %s" % str(val))

class Samples(object):
  """
  Handles loading/featurizing of chemical samples (datapoints).

  Currently knows how to load csv-files/pandas-dataframes/SDF-files.
  """

  def __init__(self, input_file, tasks, smiles_field, threshold,
               log_every_n=1000):
    """Extracts data from input as Pandas data frame"""
    rows = []
    self.tasks = tasks
    self.threshold = threshold
    self.input_file = input_file
    self.input_type = self._get_input_type(input_file)
    self.fields = self._get_fields(input_file)

    for ind, row in enumerate(self._get_raw_samples()):
      if ind % log_every_n == 0:
        print("Loading sample %d" % row_index)
      row.append(self._process_raw_sample(row))
    self.df = pd.DataFrame(rows)

  def get_samples(self):
    """Accessor for samples in this object."""
    return self.df.iterrows()

  def _get_fields(self):
    """Get the names of fields and field_types for input data."""
    # If CSV input, assume that first row contains labels
    if self.input_type == "csv":
      return self._get_raw_samples(self.input_file).next()
    elif self.input_type == "pandas":
      df = load_from_disk(self.input_file)
      return df.keys()
    elif self.input_type == "sdf":
      sample_mol = self.get_rows(self.input_file).next()
      return list(sample_mol.GetPropNames())
    else:
      raise ValueError("Unrecognized extension for %s" % self.input_file)

  def _get_input_type(self):
    """Get type of input file. Must be csv/pkl.gz/sdf file."""
    filename, file_extension = os.path.splitext(self.input_file)
    # If gzipped, need to compute extension again
    if file_extension == ".gz":
      filename, file_extension = os.path.splitext(filename)
    if file_extension == "csv":
      return "csv"
    elif file_extension == "pkl":
      return "pandas"
    elif file_extension == "sdf":
      return "sdf"
    else:
      raise ValueError("Unrecognized extension for %s" % input_file)

  def _get_raw_samples(self):
    """Returns an iterator over all rows in input_file"""
    input_type = self.get_input_type(self.input_file)
    if input_type == "csv":
      with open(self.input_file, "rb") as inp_file_obj:
        for row in csv.reader(inp_file_obj):
          if row is not None:
            yield row
    elif input_type == "pandas":
      dataframe = load_from_disk(self.input_file)
      for row in dataframe.iterrows():
        yield row
    elif input_type == "sdf":
      if ".gz" in self.input_file:
        with gzip.open(self.input_file) as inp_file_obj:
          supp = Chem.ForwardSDMolSupplier(inp_file_obj)
          for mol in supp:
            if mol is not None:
              yield mol
      else:
        with open(self.input_file) as inp_file_obj:
          supp = Chem.ForwardSDMolSupplier(inp_file_obj)
          mols = [mol for mol in supp if mol is not None]
          for mol in supp:
            if mol is not None:
              yield mol

  def _process_raw_sample(self, row):
    """Extract information from row data."""
    data = {}
    if self.input_type == "csv":
      for ind, field in enumerate(self.fields):
        data[field] = _process_field(row[ind])
      return data
    elif self.input_type == "pandas":
      # pandas rows are tuples (row_num, data)
      row = row[1]
      for field in self.fields:
        data[field] = _process_field(row[field])
    elif self.input_type == "sdf":
      mol = row
      for field in self.fields:
        if not mol.HasProp(field):
          data[field] = None
        else:
          data[field] = _process_field(mol.GetProp(field))
      data["smiles"] = Chem.MolToSmiles(mol)
    else:
      raise ValueError("Unrecognized input_type")
    if self.threshold is not None:
      for task in self.tasks:
        raw = _process_field(data[task])
        if not isinstance(raw, float):
          raise ValueError("Cannot threshold non-float fields.")
        data[field] = 1 if raw > threshold else 0
    return data

def add_vs_utils_features(df, featuretype, log_every_n=1000):
  """Generates circular fingerprints for dataset."""
  if featuretype == "fingerprints":
    featurizer = CircularFingerprint(size=1024)
  elif featuretype == "descriptors":
    featurizer = SimpleDescriptors()
  else:
    raise ValueError("Unsupported featuretype requested.")
  print("About to generate features for molecules")
  features, mol = [], None
  smiles = df["smiles"].tolist()
  for row_ind, row_data in enumerate(smiles):
    if row_ind % log_every_n == 0:
      print("Featurizing molecule %d" % row_ind)
    mol = Chem.MolFromSmiles(row_data)
    features.append(featurizer.featurize([mol]))
  df[featuretype] = features
  return

'''
Files to save:

(1) mol_id, smiles, [all feature fields] (X-ish)
(2) mol_id, smiles, split, [all task labels] (Y-ish)

'''

def standardize_df(ori_df, feature_fields, task_fields, smiles_field,
  split_field, id_field):
  df = pd.DataFrame([])
  df["mol_id"] = ori_df[[id_field]]
  df["smiles"] = ori_df[[smiles_field]]
  for task in task_fields:
    df[task] = ori_df[[task]]
  if split_field is not None:
    df["split"] = ori_df[[split_field]]

  if feature_fields is None:
    print("No feature field specified by user.")
  else:
    features_data = []
    for row in ori_df.iterrows():
      # pandas rows are tuples (row_num, row_data)
      row, feature_list = row[1], []
      for feature in feature_fields:
        feature_list.append(row[feature])
      features_data.append({"row": np.array(feature_list)})
    df["features"] = pd.DataFrame(features_data)

  return(df)

def featurize_input(self, feature_dir,
                    feature_fields, task_fields, smiles_field,
                    split_field, id_field, threshold):
  """Featurizes raw input data."""
  if len(fields) != len(field_types):
    raise ValueError("number of fields does not equal number of field types")
  if id_field is None:
    id_field = smiles_field

  df = extract_data(self.input_file, input_type, fields, field_types, task_fields,
                    smiles_field, threshold)
  print("Standardizing User DataFrame")
  df = standardize_df(df, feature_fields, task_fields, smiles_field,
                      split_field, id_field)
  print("Generating circular fingerprints")
  add_vs_utils_features(df, "fingerprints")
  print("Generating rdkit descriptors")
  add_vs_utils_features(df, "descriptors")

  print("Writing DataFrame")
  df_filename = os.path.join(
      feature_dir, "%s.joblib" %(os.path.splitext(os.path.basename(input_file))[0]))
  save_to_disk(df, df_filename)
  print("Finished saving.")

def featurize_inputs(feature_dir, input_files, input_type, fields, field_types,
                     feature_fields, task_fields, smiles_field,
                     split_field, id_field, threshold):

  featurize_input_partial = partial(featurize_input, feature_dir=feature_dir,
                                    input_type=input_type, fields=fields,
                                    field_types=field_types,
                                    feature_fields=feature_fields,
                                    task_fields=task_fields,
                                    smiles_field=smiles_field,
                                    split_field=split_field, id_field=id_field,
                                    threshold=threshold)

  pool = mp.Pool(int(mp.cpu_count()/2))
  pool.map(featurize_input_partial, input_files)
  pool.terminate()
<commit_msg>First pass of Samples class.<commit_after>"""
Process an input dataset into a format suitable for machine learning.
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
import multiprocessing as mp
import os
import cPickle as pickle
import gzip
import pandas as pd
import numpy as np
import csv
from functools import partial
from rdkit import Chem
from vs_utils.features.fingerprints import CircularFingerprint
from vs_utils.features.basic import SimpleDescriptors
from deepchem.utils.dataset import save_to_disk
from deepchem.utils.dataset import load_from_disk

def _process_field(val):
  """Parse data in a field."""
  if isinstance(val, float) or isinstance(val, np.ndarray):
    return val
  elif isinstance(val, list):
    return [process_field(elt) for elt in val]
  elif isinstance(val, str):
    try:
      return float(val)
    except ValueError:
      return val
  else:
    raise ValueError("Field of unrecognized type: %s" % str(val))

class Samples(object):
  """
  Handles loading/featurizing of chemical samples (datapoints).

  Currently knows how to load csv-files/pandas-dataframes/SDF-files.
  """

  def __init__(self, input_file, tasks, smiles_field, split_field=None,
               id_field=None, threshold=None, user_specified_features=None,
               log_every_n=1000, df=None):
    """Extracts data from input as Pandas data frame"""
    rows = []
    self.tasks = tasks
    self.user_specified_features = user_specified_features
    self.threshold = threshold
    self.input_file = input_file
    self.input_type = self._get_input_type(input_file)
    self.split_field = split_field
    self.fields = self._get_fields(input_file)
    if id_field is None:
      self.id_field = smiles_field

    if df is None:
      for ind, row in enumerate(self._get_raw_samples()):
        if ind % log_every_n == 0:
          print("Loading sample %d" % row_index)
        row.append(self._process_raw_sample(row))
      self.df = self._standardize_df(pd.DataFrame(rows))
    else:
      self.df = df

  def save(self, out):
    """
    Saves samples to disk.
    """
    sample_params = {"input_file": self.input_file,
                     "tasks": self.tasks,
                     "smiles_field": self.smiles_field,
                     "split_field": self.split_field,
                     "id_field": self.id_field,
                     "threshold": self.threshold,
                     "user_specified_features": self.user_specified_features,
                     "df": self.df}
    save_to_disk(sample_params, out)

  @staticmethod
  def load(out):
    sample_params = load_from_disk(out)
    return Samples(**sample_params)

  def get_samples(self):
    """Accessor for samples in this object."""
    return self.df.iterrows()

  def _get_fields(self):
    """Get the names of fields and field_types for input data."""
    # If CSV input, assume that first row contains labels
    if self.input_type == "csv":
      return self._get_raw_samples(self.input_file).next()
    elif self.input_type == "pandas":
      df = load_from_disk(self.input_file)
      return df.keys()
    elif self.input_type == "sdf":
      sample_mol = self.get_rows(self.input_file).next()
      return list(sample_mol.GetPropNames())
    else:
      raise ValueError("Unrecognized extension for %s" % self.input_file)

  def _get_input_type(self):
    """Get type of input file. Must be csv/pkl.gz/sdf file."""
    filename, file_extension = os.path.splitext(self.input_file)
    # If gzipped, need to compute extension again
    if file_extension == ".gz":
      filename, file_extension = os.path.splitext(filename)
    if file_extension == "csv":
      return "csv"
    elif file_extension == "pkl":
      return "pandas"
    elif file_extension == "sdf":
      return "sdf"
    else:
      raise ValueError("Unrecognized extension for %s" % input_file)

  def _get_raw_samples(self):
    """Returns an iterator over all rows in input_file"""
    input_type = self.get_input_type(self.input_file)
    if input_type == "csv":
      with open(self.input_file, "rb") as inp_file_obj:
        for row in csv.reader(inp_file_obj):
          if row is not None:
            yield row
    elif input_type == "pandas":
      dataframe = load_from_disk(self.input_file)
      for row in dataframe.iterrows():
        yield row
    elif input_type == "sdf":
      if ".gz" in self.input_file:
        with gzip.open(self.input_file) as inp_file_obj:
          supp = Chem.ForwardSDMolSupplier(inp_file_obj)
          for mol in supp:
            if mol is not None:
              yield mol
      else:
        with open(self.input_file) as inp_file_obj:
          supp = Chem.ForwardSDMolSupplier(inp_file_obj)
          mols = [mol for mol in supp if mol is not None]
          for mol in supp:
            if mol is not None:
              yield mol

  def _process_raw_sample(self, row):
    """Extract information from row data."""
    data = {}
    if self.input_type == "csv":
      for ind, field in enumerate(self.fields):
        data[field] = _process_field(row[ind])
      return data
    elif self.input_type == "pandas":
      # pandas rows are tuples (row_num, data)
      row = row[1]
      for field in self.fields:
        data[field] = _process_field(row[field])
    elif self.input_type == "sdf":
      mol = row
      for field in self.fields:
        if not mol.HasProp(field):
          data[field] = None
        else:
          data[field] = _process_field(mol.GetProp(field))
      data["smiles"] = Chem.MolToSmiles(mol)
    else:
      raise ValueError("Unrecognized input_type")
    if self.threshold is not None:
      for task in self.tasks:
        raw = _process_field(data[task])
        if not isinstance(raw, float):
          raise ValueError("Cannot threshold non-float fields.")
        data[field] = 1 if raw > threshold else 0
    return data

  def _standardize_df(self, ori_df):
    df = pd.DataFrame([])
    df["mol_id"] = ori_df[[self.id_field]]
    df["smiles"] = ori_df[[self.smiles_field]]
    for task in self.tasks:
      df[task] = ori_df[[task]]
    if self.split_field is not None:
      df["split"] = ori_df[[self.split_field]]

    if self.user_specified_features is not None:
      features_data = []
      for row in ori_df.iterrows():
        # pandas rows are tuples (row_num, row_data)
        row, feature_list = row[1], []
        for feature in user_specified_features:
          feature_list.append(row[feature])
        features_data.append({"row": np.array(feature_list)})
      df["features"] = pd.DataFrame(features_data)
    return df

  def featurize(self, featuretype, log_every_n=1000):
    """Generates circular fingerprints for dataset."""
    if featuretype == "ECFP":
      featurizer = CircularFingerprint(size=1024)
    elif featuretype == "descriptors":
      featurizer = SimpleDescriptors()
    else:
      raise ValueError("Unsupported featuretype requested.")
    print("About to generate features for molecules")
    features = []
    sample_smiles = df["smiles"].tolist()
    for ind, smiles in enumerate(sample_smiles):
      if ind % log_every_n == 0:
        print("Featurizing sample %d" % ind)
      mol = Chem.MolFromSmiles(smiles)
      features.append(featurizer.featurize([mol]))
    self.df[featuretype] = features

def featurize_input(feature_dir, input_file, user_specified_features, tasks,
                    smiles_field, split_field, id_field, threshold):
  """Featurizes raw input data."""
  samples = Samples(input_file=input_file, tasks=tasks, threshold=threshold,
                    smiles_field=smiles_field, split_field=split_field,
                    user_specified_features=user_specified_features,
                    id_field=id_field)
  print("Generating circular fingerprints")
  samples.featurize("ECFP")
  print("Generating rdkit descriptors")
  add_vs_utils_features(df, "descriptors")
  df_filename = os.path.join(
      feature_dir, "%s.joblib" %(os.path.splitext(os.path.basename(input_file))[0]))
  print("Saving samples to disk.")
  samples.save(df_filename)

def featurize_inputs(feature_dir, input_files, input_type, fields, field_types,
                     feature_fields, task_fields, smiles_field,
                     split_field, id_field, threshold):

  featurize_input_partial = partial(featurize_input, feature_dir=feature_dir,
                                    input_type=input_type, fields=fields,
                                    field_types=field_types,
                                    feature_fields=feature_fields,
                                    task_fields=task_fields,
                                    smiles_field=smiles_field,
                                    split_field=split_field, id_field=id_field,
                                    threshold=threshold)

  pool = mp.Pool(int(mp.cpu_count()/2))
  pool.map(featurize_input_partial, input_files)
  pool.terminate()
<|endoftext|>


<commit_before>'''
The road stack behaviors
'''
# Import python libs
import os
from collections import deque

# Import raet/ioflo libs
import ioflo.base.deeding
from raet.road.stacking import RoadStack
import raet.road.estating

class RaetRoadStackSetup(ioflo.base.deeding.Deed):
    '''
    Init the main road stack to use
    '''
    Ioinits = {
            'inode': 'raft.',
            'road': 'road',
            'txmsgs': {'ipath': 'txmsgs',
                       'ival': deque()},
            'rxmsgs': {'ipath': 'rxmsgs',
                       'ival': deque()},
            'opts': '.etc.opts',
            }

    def postinitio(self):
        '''
        Assign default stack values
        '''
        RoadStack.JoinentTimeout = 0.0

    def action(self):
        '''
        Setup the roadstack, only cal this once with an enter
        '''
        kind = 1
        role = self.opts.value['id']
        name = '{0}_{1}'.format(role, kind)
        main = self.opts.value.get('raet_main', True)
        mutable = self.opts.value.get('raet_mutable', True)
        always = self.opts.value.get('open_mode', True)
        mutable = mutable or always
        uid = None
        basedirpath = os.path.abspath(os.path.join(self.opts.value['cachedir'], name))
        txMsgs = self.txmsgs.value
        rxMsgs = self.rxmsgs.value
        ha = (self.opts.value['interface'], self.opts.value['port'])
        self.road.value = RoadStack(store=self.store,
                                     name=name,
                                     uid=uid,
                                     ha=ha,
                                     role=role,
                                     main=main,
                                     kind=kind,
                                     mutable=mutable,
                                     txMsgs=txMsgs,
                                     rxMsgs=rxMsgs,
                                     period=3.0,
                                     offset=0.5,
                                     auto=True,
                                     dirpath=basedirpath)

class RaetRx(ioflo.base.deeding.Deed):
    '''
    Behavior to recive inbound messages
    '''
    Ioinits = {'road': '.raft.road',
               'rxmsgs': '.raft.rxmsgs'}

    def action(self):
        self.road.value.serviceAllRx()

class RaetTx(ioflo.base.deeding.Deed):
    '''
    Behavior to send outbount messages
    '''
    Ioinits = {'road': '.raft.road'}

    def action(self):
        self.road.value.serviceAllTx()

class RaetAddRemote(ioflo.base.deeding.Deed):
    '''
    Behavior to add the remote specified in the config/cli
    '''
    Ioinits = {'road': '.raft.road',
               'opts': '.etc.opts'}

    def action(self):
        '''
        '''
        if not self.opts.value.get('remote'):
            return
        remote = self.opts.value['remote']
        comps = remote.split(':')
        ha = (comps[0], int(comps[1]))
        remote = raet.road.estating.RemoteEstate(self.road.value, fuid=0, sid=0, ha=ha)
        self.road.value.addRemote(remote)

class RaetRoadStackJoiner(ioflo.base.deeding.Deed):
    '''
    Initiates join transaction with master
    FloScript:

    do raet road stack joiner at enter
    '''
    Ioinits = {'road': '.raft.road'}

    def action(self, **kwa):
        '''
        Join with all masters
        '''
        road = self.road.value
        if road and isinstance(road, RoadStack):
            for remote in road.remotes.values():
                road.join(uid=remote.uid, timeout=0.0)

class RaetRoadStackJoined(ioflo.base.deeding.Deed):
    '''
    Updates status with .joined of zeroth remote estate (master)
    FloScript:

    do raet road stack joined
    go next if joined in .raft.status

    '''
    Ioinits = {'inode': '.raft',
               'road': 'road',
               'status': {'ipath': 'status', 'ival': {'joined': False,
                                                      'allowed': False,
                                                      'alived': False,
                                                      'rejected': False,
                                                      'idle': False,}}}

    def action(self, **kwa):
        '''
        Update .status share
        '''
        road = self.road.value
        joined = False
        if road and isinstance(road, RoadStack):
            if road.remotes:
                for remote in road.remotes.values():
                    joined = any([remote.joined for remote in road.remotes.values()])
        self.status.update(joined=joined)

class RaetRoadStackAllower(ioflo.base.deeding.Deed):
    '''
    Initiates allow (CurveCP handshake) transaction with master
    FloScript:

    do raet road stack allower at enter

    '''
    Ioinits = {
            'inode': '.raft',
            'road': 'road'}

    def action(self, **kwa):
        '''
        Receive any udp packets on server socket and put in rxes
        Send any packets in txes
        '''
        road = self.road.value
        if road and isinstance(road, RoadStack):
            road.allow(timeout=0.0)
        return None

class RaetRoadStackAllowed(ioflo.base.deeding.Deed):
    '''
    Updates status with .allowed of zeroth remote estate (master)
    FloScript:

    do raet road stack allowed
    go next if allowed in .raft.status

    '''
    Ioinits = {
            'inode': '.raft',
            'stack': 'road',
            'status': {'ipath': 'status', 'ival': {'joined': False,
                                                   'allowed': False,
                                                   'alived': False,
                                                   'rejected': False,
                                                   'idle': False}}}

    def action(self, **kwa):
        '''
        Update .status share
        '''
        road = self.road.value
        allowed = False
        if road and isinstance(road, RoadStack):
            if road.remotes:
                for remote in road.remotes.values():
                    allowed = any([remote.allowed for remote in road.remotes.values()])
        self.status.update(allowed=allowed)

class RaetRoadStackStarted(ioflo.base.deeding.Deed):
    '''
    Send a started ping to all remotes
    '''
    Ioinits = {
            'road': '.raft.road',
            }

    def action(self):
        '''
        Send an initial message on the connection
        '''
        for remote in self.road.value.remotes.values():
            self.road.value.message({'Started': True}, remote.uid)
<commit_msg>finish namespaceing refactor<commit_after>'''
The road stack behaviors
'''
# Import python libs
import os
from collections import deque

# Import raet/ioflo libs
import ioflo.base.deeding
from raet.road.stacking import RoadStack
import raet.road.estating

class RaetRoadStackSetup(ioflo.base.deeding.Deed):
    '''
    Init the main road stack to use
    '''
    Ioinits = {
            'inode': 'raft.',
            'road': 'road',
            'txmsgs': {'ipath': 'txmsgs',
                       'ival': deque()},
            'rxmsgs': {'ipath': 'rxmsgs',
                       'ival': deque()},
            'opts': '.etc.opts',
            }

    def postinitio(self):
        '''
        Assign default stack values
        '''
        RoadStack.JoinentTimeout = 0.0

    def action(self):
        '''
        Setup the roadstack, only cal this once with an enter
        '''
        kind = 1
        role = self.opts.value['id']
        name = '{0}_{1}'.format(role, kind)
        main = self.opts.value.get('raet_main', True)
        mutable = self.opts.value.get('raet_mutable', True)
        always = self.opts.value.get('open_mode', True)
        mutable = mutable or always
        uid = None
        basedirpath = os.path.abspath(os.path.join(self.opts.value['cachedir'], name))
        txMsgs = self.txmsgs.value
        rxMsgs = self.rxmsgs.value
        ha = (self.opts.value['interface'], self.opts.value['port'])
        self.road.value = RoadStack(store=self.store,
                                     name=name,
                                     uid=uid,
                                     ha=ha,
                                     role=role,
                                     main=main,
                                     kind=kind,
                                     mutable=mutable,
                                     txMsgs=txMsgs,
                                     rxMsgs=rxMsgs,
                                     period=3.0,
                                     offset=0.5,
                                     auto=True,
                                     dirpath=basedirpath)

class RaetRx(ioflo.base.deeding.Deed):
    '''
    Behavior to recive inbound messages
    '''
    Ioinits = {'road': '.raft.road',
               'rxmsgs': '.raft.rxmsgs'}

    def action(self):
        self.road.value.serviceAllRx()

class RaetTx(ioflo.base.deeding.Deed):
    '''
    Behavior to send outbount messages
    '''
    Ioinits = {'road': '.raft.road'}

    def action(self):
        self.road.value.serviceAllTx()

class RaetAddRemote(ioflo.base.deeding.Deed):
    '''
    Behavior to add the remote specified in the config/cli
    '''
    Ioinits = {'road': '.raft.road',
               'opts': '.etc.opts'}

    def action(self):
        '''
        '''
        if not self.opts.value.get('remote'):
            return
        remote = self.opts.value['remote']
        comps = remote.split(':')
        ha = (comps[0], int(comps[1]))
        remote = raet.road.estating.RemoteEstate(self.road.value, fuid=0, sid=0, ha=ha)
        self.road.value.addRemote(remote)

class RaetRoadStackJoiner(ioflo.base.deeding.Deed):
    '''
    Initiates join transaction with master
    FloScript:

    do raet road stack joiner at enter
    '''
    Ioinits = {'road': '.raft.road'}

    def action(self, **kwa):
        '''
        Join with all masters
        '''
        road = self.road.value
        if road and isinstance(road, RoadStack):
            for remote in road.remotes.values():
                road.join(uid=remote.uid, timeout=0.0)

class RaetRoadStackJoined(ioflo.base.deeding.Deed):
    '''
    Updates status with .joined of zeroth remote estate (master)
    FloScript:

    do raet road stack joined
    go next if joined in .raft.status

    '''
    Ioinits = {'inode': '.raft',
               'road': 'road',
               'status': {'ipath': 'status', 'ival': {'joined': False,
                                                      'allowed': False,
                                                      'alived': False,
                                                      'rejected': False,
                                                      'idle': False,}}}

    def action(self, **kwa):
        '''
        Update .status share
        '''
        road = self.road.value
        joined = False
        if road and isinstance(road, RoadStack):
            if road.remotes:
                for remote in road.remotes.values():
                    joined = any([remote.joined for remote in road.remotes.values()])
        self.status.update(joined=joined)

class RaetRoadStackAllower(ioflo.base.deeding.Deed):
    '''
    Initiates allow (CurveCP handshake) transaction with master
    FloScript:

    do raet road stack allower at enter

    '''
    Ioinits = {
            'inode': '.raft',
            'road': 'road'}

    def action(self, **kwa):
        '''
        Receive any udp packets on server socket and put in rxes
        Send any packets in txes
        '''
        road = self.road.value
        if road and isinstance(road, RoadStack):
            road.allow(timeout=0.0)
        return None

class RaetRoadStackAllowed(ioflo.base.deeding.Deed):
    '''
    Updates status with .allowed of zeroth remote estate (master)
    FloScript:

    do raet road stack allowed
    go next if allowed in .raft.status

    '''
    Ioinits = {
            'inode': '.raft',
            'road': 'road',
            'status': {'ipath': 'status', 'ival': {'joined': False,
                                                   'allowed': False,
                                                   'alived': False,
                                                   'rejected': False,
                                                   'idle': False}}}

    def action(self, **kwa):
        '''
        Update .status share
        '''
        road = self.road.value
        allowed = False
        if road and isinstance(road, RoadStack):
            if road.remotes:
                for remote in road.remotes.values():
                    allowed = any([remote.allowed for remote in road.remotes.values()])
        self.status.update(allowed=allowed)

class RaetRoadStackStarted(ioflo.base.deeding.Deed):
    '''
    Send a started ping to all remotes
    '''
    Ioinits = {
            'road': '.raft.road',
            }

    def action(self):
        '''
        Send an initial message on the connection
        '''
        for remote in self.road.value.remotes.values():
            self.road.value.message({'Started': True}, remote.uid)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Since counts are in the form of bedgraph, counts as a result of this script
are unstranded. Any input should be strand specific.
"""
import os
import tempfile
import subprocess as sp
# from cruzdb import Genome

# def get_sizes(species):
#     tmp = tempfile.mkstemp(suffix=".bed")[1]
#     g = Genome(db=species)
#     df = g.dataframe("chromInfo")
#     df.to_csv(tmp, cols=['chrom','sizes'],
#                 sep="\t", header=False, index=False)
#     return tmp

def add_slop(bed, sizes, n):
    """add slop onto polya sites."""
    tmp = tempfile.mkstemp(suffix=".bed")[1]
    cmd = "bedtools slop -b %d -i %s -g %s |\
            bedtools sort -i - > %s" % (n, bed, sizes, tmp)
    sp.call(cmd, shell=True)
    return tmp

def map_counts(bed, bedgraph):
    """print counts to stdout using bedtools map"""
    # overlap and get counts
    # add appropriate gene name
    # add indeces for sorting
    # sort by gene name then site number
    # output dexseq columns
    cmd = """bedtools map -c 4 -o max -null 0 -a %s -b %s |\
              awk '{split($4, symbol, "|"); split(symbol[1], gene, ".");print gene[3]":"$4"\t"$7}' |\
              awk '{split($1, full, ":"); split(full[2], site, "."); print full[1]"\t"site[4]"\t"$0}' |\
              sort -k1,1 -k2,2n |\
              awk '{print $3"\t"$4}'"""\
              % (bed, bedgraph)
    sp.call(cmd, shell=True)

def main(counts, sites, sizes, bases):
    # tmpsizes = get_sizes(species)
    tmp = add_slop(sites, sizes, bases)
    map_counts(tmp, counts)
    os.remove(tmp)

if __name__ == '__main__':
    import argparse
    p = argparse.ArgumentParser(description=__doc__,
            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument("counts", help="read counts for a sample as bedgraph")
    p.add_argument("sites", help="merged polya sites as bed")
    p.add_argument("sizes", help="chromosome sizes for specific genome")
    # p.add_argument("-s", dest="species", default="hg19",
    #         help="ucsc table to use")
    p.add_argument("-b", dest="bases", type=int, default=2,
            help="increase region -b base pairs in each direction when \
            finding max read counts")
    args = p.parse_args()
    main(args.counts, args.sites, args.sizes, args.bases)
<commit_msg>started obtaining counts differently<commit_after><|endoftext|>