<commit_before>import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms, datasets
import logging
import argparse
import sys
import asyncio

FORMAT = "%(asctime)s %(levelname)s %(filename)s(l:%(lineno)d) - %(message)s"
LOG_LEVEL = logging.INFO
logging.basicConfig(format=FORMAT, level=LOG_LEVEL)

import syft as sy

from syft import workers
from syft.frameworks.torch import pointers
from syft.frameworks.torch.federated import utils

logger = logging.getLogger(__name__)

LOG_INTERVAL = 25

# Loss function
@torch.jit.script
def loss_fn(output, target):
    return F.nll_loss(output, target)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4 * 4 * 50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4 * 4 * 50)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

def define_and_get_arguments(args=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Run federated learning using websocket client workers."
    )
    parser.add_argument("--batch_size", type=int, default=32, help="batch size of the training")
    parser.add_argument(
        "--test_batch_size", type=int, default=32, help="batch size used for the test data"
    )
    parser.add_argument("--epochs", type=int, default=10, help="number of epochs to train")
    parser.add_argument(
        "--federate_after_n_batches",
        type=int,
        default=50,
        help="number of training steps performed on each remote worker before averaging",
    )
    parser.add_argument("--lr", type=float, default=0.01, help="learning rate")
    parser.add_argument("--cuda", action="store_true", help="use cuda")
    parser.add_argument("--seed", type=int, default=1, help="seed used for randomization")
    parser.add_argument("--save_model", action="store_true", help="if set, model will be saved")
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help="if set, websocket client workers will be started in verbose mode",
    )
    parser.add_argument(
        "--use_virtual", action="store_true", help="if set, virtual workers will be used"
    )

    args = parser.parse_args(args=args)
    return args

def accuracy(pred_softmax, target):
    nr_elems = len(target)
    pred = pred_softmax.argmax(dim=1)
    logger.debug("predicted: %s", pred)
    logger.debug("target:    %s", target)
    return (pred == target).sum().numpy() / float(nr_elems)

async def fit_model_on_worker(worker, traced_model, batch_size, curr_epoch, max_nr_batches):
    # Create and send train config
    train_config = sy.TrainConfig(
        batch_size=batch_size, shuffle=True, max_nr_batches=max_nr_batches, epochs=1
    )
    train_config.send(worker, traced_model=traced_model, traced_loss_fn=loss_fn)
    logger.info("Training round %s, calling fit on worker: %s", curr_epoch, worker.id)
    loss = await worker.fit(dataset="mnist", return_ids=[0])
    logger.info("Training round: %s, worker: %s, avg_loss: %s", curr_epoch, worker.id, loss.mean())
    # logger.debug("Worker state: %s", worker)
    # logger.debug("Worker objects: \n%s", worker.list_objects_remote())
    # logger.debug("loss: mean %s, max %s, min %s", loss.mean(), loss.max(), loss.min())
    model = None
    if not torch.isnan(loss).any():
        model = train_config.model_ptr.get().obj
    return worker.id, model, loss

async def main():
    args = define_and_get_arguments()
    # args.use_virtual = True

    hook = sy.TorchHook(torch)
    me = hook.local_worker

    if args.use_virtual:
        alice = sy.workers.VirtualWorker(id="alice", hook=hook, verbose=args.verbose)
        bob = sy.workers.VirtualWorker(id="bob", hook=hook, verbose=args.verbose)
        charlie = sy.workers.VirtualWorker(id="charlie", hook=hook, verbose=args.verbose)

        mnist_trainset = datasets.MNIST(
            root="./data",
            train=True,
            download=True,
            transform=transforms.Compose(
                [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
            ),
        )

        dataset = sy.BaseDataset(
            data=mnist_trainset.train_data,
            targets=mnist_trainset.train_labels,
            transform=mnist_trainset.transform,
        )
        # Note, that using virtual workers, all workers have all numbers.
        alice.add_dataset(dataset, key="mnist")
        bob.add_dataset(dataset, key="mnist")
        charlie.add_dataset(dataset, key="mnist")
    else:
        kwargs_websocket = {"host": "localhost", "hook": hook, "verbose": args.verbose}
        alice = sy.workers.WebsocketClientWorker(id="alice", port=8777, **kwargs_websocket)
        bob = sy.workers.WebsocketClientWorker(id="bob", port=8778, **kwargs_websocket)
        charlie = sy.workers.WebsocketClientWorker(id="charlie", port=8779, **kwargs_websocket)

    worker_instances = [alice, bob, charlie]

    use_cuda = args.cuda and torch.cuda.is_available()

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {}

    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST(
            "../data",
            train=True,
            transform=transforms.Compose(
                [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
            ),
        ),
        batch_size=args.test_batch_size,
        shuffle=True,
        **kwargs,
    )

    model = Net().to(device)

    (data, target) = test_loader.__iter__().next()
    traced_model = torch.jit.trace(model, data)

    for epoch in range(1, args.epochs + 1):
        logger.info("Starting epoch %s/%s", epoch, args.epochs)

        results = await asyncio.gather(
            *[
                fit_model_on_worker(
                    worker=worker,
                    traced_model=traced_model,
                    batch_size=args.batch_size,
                    curr_epoch=epoch,
                    max_nr_batches=args.federate_after_n_batches,
                )
                for worker in worker_instances
            ]
        )
        models = {}
        loss_values = {}
        for worker_id, worker_model, worker_loss in results:
            if worker_model is not None:
                models[worker_id] = worker_model
                loss_values[worker_id] = worker_loss

        traced_model = utils.federated_avg(models)
        data, target = test_loader.__iter__().next()
        pred_test = traced_model(data)
        loss = loss_fn(output=pred_test, target=target)
        logger.info("Test dataset: Loss: %s, accuracy = %s", loss, accuracy(pred_test, target))

    if args.save_model:
        torch.save(model.state_dict(), "mnist_cnn.pt")

if __name__ == "__main__":
    # FORMAT = "%(asctime)s %(levelname)s %(filename)s(l:%(lineno)d) - %(message)s"
    # LOG_LEVEL = logging.DEBUG
    # logging.basicConfig(format=FORMAT, level=LOG_LEVEL)

    websockets_logger = logging.getLogger("websockets")
    websockets_logger.setLevel(logging.INFO)
    websockets_logger.addHandler(logging.StreamHandler())

    asyncio.get_event_loop().run_until_complete(main())
<commit_msg>Correct import usage<commit_after>import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms, datasets
import logging
import argparse
import sys
import asyncio

FORMAT = "%(asctime)s %(levelname)s %(filename)s(l:%(lineno)d) - %(message)s"
LOG_LEVEL = logging.INFO
logging.basicConfig(format=FORMAT, level=LOG_LEVEL)

import syft as sy

from syft import workers
from syft.frameworks.torch.federated import utils

logger = logging.getLogger(__name__)

LOG_INTERVAL = 25

# Loss function
@torch.jit.script
def loss_fn(output, target):
    return F.nll_loss(output, target)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4 * 4 * 50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4 * 4 * 50)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

def define_and_get_arguments(args=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Run federated learning using websocket client workers."
    )
    parser.add_argument("--batch_size", type=int, default=32, help="batch size of the training")
    parser.add_argument(
        "--test_batch_size", type=int, default=32, help="batch size used for the test data"
    )
    parser.add_argument("--epochs", type=int, default=10, help="number of epochs to train")
    parser.add_argument(
        "--federate_after_n_batches",
        type=int,
        default=50,
        help="number of training steps performed on each remote worker before averaging",
    )
    parser.add_argument("--lr", type=float, default=0.01, help="learning rate")
    parser.add_argument("--cuda", action="store_true", help="use cuda")
    parser.add_argument("--seed", type=int, default=1, help="seed used for randomization")
    parser.add_argument("--save_model", action="store_true", help="if set, model will be saved")
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help="if set, websocket client workers will be started in verbose mode",
    )
    parser.add_argument(
        "--use_virtual", action="store_true", help="if set, virtual workers will be used"
    )

    args = parser.parse_args(args=args)
    return args

def accuracy(pred_softmax, target):
    nr_elems = len(target)
    pred = pred_softmax.argmax(dim=1)
    logger.debug("predicted: %s", pred)
    logger.debug("target:    %s", target)
    return (pred == target).sum().numpy() / float(nr_elems)

async def fit_model_on_worker(worker, traced_model, batch_size, curr_epoch, max_nr_batches):
    # Create and send train config
    train_config = sy.TrainConfig(
        batch_size=batch_size, shuffle=True, max_nr_batches=max_nr_batches, epochs=1
    )
    train_config.send(worker, traced_model=traced_model, traced_loss_fn=loss_fn)
    logger.info("Training round %s, calling fit on worker: %s", curr_epoch, worker.id)
    loss = await worker.fit(dataset="mnist", return_ids=[0])
    logger.info("Training round: %s, worker: %s, avg_loss: %s", curr_epoch, worker.id, loss.mean())
    # logger.debug("Worker state: %s", worker)
    # logger.debug("Worker objects: \n%s", worker.list_objects_remote())
    # logger.debug("loss: mean %s, max %s, min %s", loss.mean(), loss.max(), loss.min())
    model = None
    if not torch.isnan(loss).any():
        model = train_config.model_ptr.get().obj
    return worker.id, model, loss

async def main():
    args = define_and_get_arguments()
    # args.use_virtual = True

    hook = sy.TorchHook(torch)
    me = hook.local_worker

    if args.use_virtual:
        alice = workers.VirtualWorker(id="alice", hook=hook, verbose=args.verbose)
        bob = workers.VirtualWorker(id="bob", hook=hook, verbose=args.verbose)
        charlie = workers.VirtualWorker(id="charlie", hook=hook, verbose=args.verbose)

        mnist_trainset = datasets.MNIST(
            root="./data",
            train=True,
            download=True,
            transform=transforms.Compose(
                [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
            ),
        )

        dataset = sy.BaseDataset(
            data=mnist_trainset.train_data,
            targets=mnist_trainset.train_labels,
            transform=mnist_trainset.transform,
        )
        # Note, that using virtual workers, all workers have all numbers.
        alice.add_dataset(dataset, key="mnist")
        bob.add_dataset(dataset, key="mnist")
        charlie.add_dataset(dataset, key="mnist")
    else:
        kwargs_websocket = {"host": "localhost", "hook": hook, "verbose": args.verbose}
        alice = workers.WebsocketClientWorker(id="alice", port=8777, **kwargs_websocket)
        bob = workers.WebsocketClientWorker(id="bob", port=8778, **kwargs_websocket)
        charlie = workers.WebsocketClientWorker(id="charlie", port=8779, **kwargs_websocket)

    worker_instances = [alice, bob, charlie]

    use_cuda = args.cuda and torch.cuda.is_available()

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {}

    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST(
            "../data",
            train=True,
            transform=transforms.Compose(
                [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
            ),
        ),
        batch_size=args.test_batch_size,
        shuffle=True,
        **kwargs,
    )

    model = Net().to(device)

    (data, target) = test_loader.__iter__().next()
    traced_model = torch.jit.trace(model, data)

    for epoch in range(1, args.epochs + 1):
        logger.info("Starting epoch %s/%s", epoch, args.epochs)

        results = await asyncio.gather(
            *[
                fit_model_on_worker(
                    worker=worker,
                    traced_model=traced_model,
                    batch_size=args.batch_size,
                    curr_epoch=epoch,
                    max_nr_batches=args.federate_after_n_batches,
                )
                for worker in worker_instances
            ]
        )
        models = {}
        loss_values = {}
        for worker_id, worker_model, worker_loss in results:
            if worker_model is not None:
                models[worker_id] = worker_model
                loss_values[worker_id] = worker_loss

        traced_model = utils.federated_avg(models)
        data, target = test_loader.__iter__().next()
        pred_test = traced_model(data)
        loss = loss_fn(output=pred_test, target=target)
        logger.info("Test dataset: Loss: %s, accuracy = %s", loss, accuracy(pred_test, target))

    if args.save_model:
        torch.save(model.state_dict(), "mnist_cnn.pt")

if __name__ == "__main__":
    # FORMAT = "%(asctime)s %(levelname)s %(filename)s(l:%(lineno)d) - %(message)s"
    # LOG_LEVEL = logging.DEBUG
    # logging.basicConfig(format=FORMAT, level=LOG_LEVEL)

    websockets_logger = logging.getLogger("websockets")
    websockets_logger.setLevel(logging.INFO)
    websockets_logger.addHandler(logging.StreamHandler())

    asyncio.get_event_loop().run_until_complete(main())
<|endoftext|>


<commit_before>
class Value():
    """
    Wrap around hdl value with overloaded operators
    http://www.rafekettler.com/magicmethods.html

    operators are overloaded in every type separately
    """
    def __init__(self, val, _type, vldMask, updateTime=-1):
        if isinstance(val, Value):
            val = val.val

        self.val = val
        self._dtype = _type
        self.vldMask = vldMask
        self.updateTime = updateTime

    def _convert(self, toT):
        return self._dtype.convert(self, toT)

    def staticEval(self):
        return self.clone()

    def simEval(self, simulator):
        return self.clone()

    def clone(self):
        return self.__class__(self.val, self._dtype, self.vldMask, self.updateTime)

    def __hash__(self):
        return hash((self._dtype, self.val, self.vldMask, self.updateTime))

    def __repr__(self):
        return "<Value {0:s}, vldMask {1:b}, updateTime {2:.2f}>".format(
                    str(self.val), self.vldMask, self.updateTime)

    @classmethod
    def fromPy(cls, val, typeObj):
        raise NotImplementedError("fromPy fn is not implemented for %s" % (str(cls)))

    def __nonzero__(self):
        raise NotImplementedError()
    def __bool__(self):
        raise NotImplementedError()
    def __pos__(self):
        raise NotImplementedError()
    def __neg__(self):
        raise NotImplementedError()
    def __abs__(self):
        raise NotImplementedError()
    def __invert__(self):
        raise NotImplementedError()
    def __round__(self, n):
        raise NotImplementedError()
    def __floor__(self):
        raise NotImplementedError()
    def __ceil__(self):
        raise NotImplementedError()
    def __trunc__(self):
        raise NotImplementedError()
    def __add__(self, other):
        raise NotImplementedError()
    def __sub__(self, other):
        raise NotImplementedError()
    def __mul__(self, other):
        raise NotImplementedError()
    def __floordiv__(self, other):
        raise NotImplementedError()
    def __div__(self, other):
        raise NotImplementedError()
    def __truediv__(self, other):
        raise NotImplementedError()
    def __mod__(self, other):
        raise NotImplementedError()
    def __divmod__(self, other):
        raise NotImplementedError()
    def __pow__(self, other):
        raise NotImplementedError()
    def __lshift__(self, other):
        raise NotImplementedError()
    def __rshift__(self, other):
        raise NotImplementedError()
    def __and__(self, other):
        raise NotImplementedError()
    def __or__(self, other):
        raise NotImplementedError()
    def __xor__(self, other):
        raise NotImplementedError()
    def __eq__(self, other):
        if areValues(self, other):
            return self._dtype == other._dtype and  bool(self._eq(other))
        else:
            super().__eq__(other)

    def _eq(self, other):
        raise NotImplementedError()
    def __ne__(self, other):
        eq = self._eq(other)
        eq.val = not eq.val
        return eq
    def _concat(self, other):
        raise NotImplementedError()
    def __lt__(self, other):
        raise NotImplementedError()
    def __le__(self, other):
        raise NotImplementedError()
    def __gt__(self, other):
        raise NotImplementedError()
    def __ge__(self, other):
        raise NotImplementedError()

    def _hasEvent(self, now):
        raise NotImplementedError()

    def _onRisingEdge(self, now):
        raise NotImplementedError()

def areValues(*items):
    res = True
    for i in items:
        res = res and isinstance(i, Value)
    return res
<commit_msg>new: _isFullVld()<commit_after>
class Value():
    """
    Wrap around hdl value with overloaded operators
    http://www.rafekettler.com/magicmethods.html

    operators are overloaded in every type separately
    """
    def __init__(self, val, _type, vldMask, updateTime=-1):
        if isinstance(val, Value):
            val = val.val

        self.val = val
        self._dtype = _type
        self.vldMask = vldMask
        self.updateTime = updateTime

    def _isFullVld(self):
        return self.vldMask == self._dtype.all_mask()

    def _convert(self, toT):
        return self._dtype.convert(self, toT)

    def staticEval(self):
        return self.clone()

    def simEval(self, simulator):
        return self.clone()

    def clone(self):
        return self.__class__(self.val, self._dtype, self.vldMask, self.updateTime)

    def __hash__(self):
        return hash((self._dtype, self.val, self.vldMask, self.updateTime))

    def __repr__(self):
        return "<Value {0:s}, vldMask {1:b}, updateTime {2:.2f}>".format(
                    str(self.val), self.vldMask, self.updateTime)

    @classmethod
    def fromPy(cls, val, typeObj):
        raise NotImplementedError("fromPy fn is not implemented for %s" % (str(cls)))

    def __nonzero__(self):
        raise NotImplementedError()
    def __bool__(self):
        raise NotImplementedError()
    def __pos__(self):
        raise NotImplementedError()
    def __neg__(self):
        raise NotImplementedError()
    def __abs__(self):
        raise NotImplementedError()
    def __invert__(self):
        raise NotImplementedError()
    def __round__(self, n):
        raise NotImplementedError()
    def __floor__(self):
        raise NotImplementedError()
    def __ceil__(self):
        raise NotImplementedError()
    def __trunc__(self):
        raise NotImplementedError()
    def __add__(self, other):
        raise NotImplementedError()
    def __sub__(self, other):
        raise NotImplementedError()
    def __mul__(self, other):
        raise NotImplementedError()
    def __floordiv__(self, other):
        raise NotImplementedError()
    def __div__(self, other):
        raise NotImplementedError()
    def __truediv__(self, other):
        raise NotImplementedError()
    def __mod__(self, other):
        raise NotImplementedError()
    def __divmod__(self, other):
        raise NotImplementedError()
    def __pow__(self, other):
        raise NotImplementedError()
    def __lshift__(self, other):
        raise NotImplementedError()
    def __rshift__(self, other):
        raise NotImplementedError()
    def __and__(self, other):
        raise NotImplementedError()
    def __or__(self, other):
        raise NotImplementedError()
    def __xor__(self, other):
        raise NotImplementedError()
    def __eq__(self, other):
        if areValues(self, other):
            return self._dtype == other._dtype and  bool(self._eq(other))
        else:
            super().__eq__(other)

    def _eq(self, other):
        raise NotImplementedError()
    def __ne__(self, other):
        eq = self._eq(other)
        eq.val = not eq.val
        return eq
    def _concat(self, other):
        raise NotImplementedError()
    def __lt__(self, other):
        raise NotImplementedError()
    def __le__(self, other):
        raise NotImplementedError()
    def __gt__(self, other):
        raise NotImplementedError()
    def __ge__(self, other):
        raise NotImplementedError()

    def _hasEvent(self, now):
        raise NotImplementedError()

    def _onRisingEdge(self, now):
        raise NotImplementedError()

def areValues(*items):
    res = True
    for i in items:
        res = res and isinstance(i, Value)
    return res
<|endoftext|>


<commit_before>import datetime
import requests
import json
import re
from django.conf import settings
from registrations.models import Source

session = requests.Session()
http = requests.adapters.HTTPAdapter(max_retries=5)
https = requests.adapters.HTTPAdapter(max_retries=5)
session.mount('http://', http)
session.mount('https://', https)

def get_today():
    return datetime.datetime.today()

def calc_pregnancy_week_lmp(today, lmp):
    """ Calculate how far along the mother's prenancy is in weeks.
    """
    last_period_date = datetime.datetime.strptime(lmp, "%Y%m%d")
    time_diff = today - last_period_date
    preg_weeks = int(time_diff.days / 7)
    # You can't be one week pregnant (smaller numbers will be rejected)
    if preg_weeks == 1:
        preg_weeks = 2
    return preg_weeks

def calc_baby_age(today, baby_dob):
    """ Calculate the baby's age in weeks.
    """
    baby_dob_date = datetime.datetime.strptime(baby_dob, "%Y%m%d")
    time_diff = today - baby_dob_date
    if time_diff.days >= 0:
        age_weeks = int(time_diff.days / 7)
        return age_weeks
    else:
        # Return -1 if the date is in the future
        return -1

def get_identity(identity):
    url = "%s/%s/%s/" % (settings.IDENTITY_STORE_URL, "identities", identity)
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.get(url, headers=headers)
    return r.json()

def get_identity_address(identity):
    url = "%s/%s/%s/addresses/msisdn" % (settings.IDENTITY_STORE_URL,
                                         "identities", identity)
    params = {"default": True}
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.get(url, params=params, headers=headers).json()
    if len(r["results"]) > 0:
        return r["results"][0]["address"]
    else:
        return None

def search_identities(params):
    """ Returns the identities matching the given parameters
    """
    url = "%s/%s/search/" % (settings.IDENTITY_STORE_URL, "identities")
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, params=params, headers=headers)
    r.raise_for_status()
    r = r.json()

    while True:
        for identity in r.get('results', []):
            yield identity
        if r.get('next'):
            r = session.get(r['next'], headers=headers)
            r.raise_for_status()
            r = r.json()
        else:
            break

def patch_identity(identity, data):
    """ Patches the given identity with the data provided
    """
    url = "%s/%s/%s/" % (settings.IDENTITY_STORE_URL, "identities", identity)
    data = data
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.patch(url, data=json.dumps(data), headers=headers)
    r.raise_for_status()
    return r.json()

def get_messageset_by_shortname(short_name):
    url = "%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL, "messageset")
    params = {'short_name': short_name}
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.get(url, params=params, headers=headers)
    return r.json()["results"][0]  # messagesets should be unique, return 1st

def get_messageset(messageset_id):
    url = "%s/%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL, "messageset",
                         messageset_id)
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.get(url, headers=headers)
    return r.json()

def get_schedule(schedule_id):
    url = "%s/%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL,
                         "schedule", schedule_id)
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.get(url, headers=headers)
    return r.json()

def get_subscriptions(identity):
    """ Gets the active subscriptions for an identity
    """
    url = "%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL, "subscriptions")
    params = {'id': identity, 'active': True}
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.get(url, params=params, headers=headers)
    return r.json()["results"]

def patch_subscription(subscription, data):
    """ Patches the given subscription with the data provided
    """
    url = "%s/%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL,
                         "subscriptions", subscription["id"])
    data = data
    headers = {
        'Authorization':
        'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = requests.patch(url, data=data, headers=headers)
    return r.json()

def deactivate_subscription(subscription):
    """ Sets a subscription deactive via a Patch request
    """
    return patch_subscription(subscription, {"active": False})

def get_messageset_short_name(stage, recipient, msg_type, weeks, voice_days,
                              voice_times):

    if recipient == "household":
        msg_type = "audio"

    if stage == "prebirth":
        week_range = "10_42"
    elif stage == "miscarriage":
        week_range = "0_2"
    elif stage == "postbirth":
        if recipient == "household":
            week_range = "0_52"
        elif 0 <= weeks <= 12:
            week_range = "0_12"
        elif 13 <= weeks <= 52:
            week_range = "13_52"

    if msg_type == "text":
        short_name = "%s.%s.%s.%s" % (
            stage, recipient, msg_type, week_range)
    else:
        short_name = "%s.%s.%s.%s.%s.%s" % (
            stage, recipient, msg_type, week_range, voice_days, voice_times)

    return short_name

def get_messageset_schedule_sequence(short_name, weeks):
    # get messageset
    messageset = get_messageset_by_shortname(short_name)

    messageset_id = messageset["id"]
    schedule_id = messageset["default_schedule"]
    # get schedule
    schedule = get_schedule(schedule_id)

    # calculate next_sequence_number
    # get schedule days of week: comma-seperated str e.g. '1,3' for Mon & Wed
    days_of_week = schedule["day_of_week"]
    # determine how many times a week messages are sent e.g. 2 for '1,3'
    msgs_per_week = len(days_of_week.split(','))
    # determine starting message
    # check if in prebirth stage - only starting messaging in week 10
    if 'miscarriage' in short_name:
        next_sequence_number = 1  # always start loss messages at 1
    elif 'prebirth' in short_name:
        next_sequence_number = msgs_per_week * (
            weeks - settings.PREBIRTH_MIN_WEEKS)
        if next_sequence_number == 0:
            next_sequence_number = 1  # next_sequence_number cannot be 0
    elif '13_52' in short_name:
        next_sequence_number = msgs_per_week * (weeks - 13)
        if next_sequence_number == 0:
            next_sequence_number = 1  # next_sequence_number cannot be 0
    else:
        next_sequence_number = msgs_per_week * weeks
        if next_sequence_number == 0:
            next_sequence_number = 1  # next_sequence_number cannot be 0

    return (messageset_id, schedule_id, next_sequence_number)

def post_message(payload):
    result = requests.post(
        url="%s/outbound/" % settings.MESSAGE_SENDER_URL,
        data=json.dumps(payload),
        headers={
            'Content-Type': 'application/json',
            'Authorization': 'Token %s' % (
                settings.MESSAGE_SENDER_TOKEN,)
        }
    ).json()
    return result

def get_available_metrics():
    available_metrics = []
    available_metrics.extend(settings.METRICS_REALTIME)
    available_metrics.extend(settings.METRICS_SCHEDULED)

    sources = Source.objects.all()
    for source in sources:
        # only append usernames with characters that are all alphanumeric
        # and/or underscores
        if re.match(r'\w+$', source.user.username):
            available_metrics.append(
                "registrations.source.%s.sum" % source.user.username)

    return available_metrics

def normalise_string(string):
    """ Strips trailing whitespace from string, lowercases it and replaces
        spaces with underscores
    """
    string = (string.strip()).lower()
    return re.sub(r'\W+', '_', string)

def timestamp_to_epoch(timestamp):
    """
    Takes a timestamp and returns a float representing the unix epoch time.
    """
    return (timestamp - datetime.datetime.utcfromtimestamp(0)).total_seconds()
<commit_msg>Add http retries for all calls in the utils<commit_after>import datetime
import requests
import json
import re
from django.conf import settings
from registrations.models import Source

session = requests.Session()
http = requests.adapters.HTTPAdapter(max_retries=5)
https = requests.adapters.HTTPAdapter(max_retries=5)
session.mount('http://', http)
session.mount('https://', https)

def get_today():
    return datetime.datetime.today()

def calc_pregnancy_week_lmp(today, lmp):
    """ Calculate how far along the mother's prenancy is in weeks.
    """
    last_period_date = datetime.datetime.strptime(lmp, "%Y%m%d")
    time_diff = today - last_period_date
    preg_weeks = int(time_diff.days / 7)
    # You can't be one week pregnant (smaller numbers will be rejected)
    if preg_weeks == 1:
        preg_weeks = 2
    return preg_weeks

def calc_baby_age(today, baby_dob):
    """ Calculate the baby's age in weeks.
    """
    baby_dob_date = datetime.datetime.strptime(baby_dob, "%Y%m%d")
    time_diff = today - baby_dob_date
    if time_diff.days >= 0:
        age_weeks = int(time_diff.days / 7)
        return age_weeks
    else:
        # Return -1 if the date is in the future
        return -1

def get_identity(identity):
    url = "%s/%s/%s/" % (settings.IDENTITY_STORE_URL, "identities", identity)
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, headers=headers)
    return r.json()

def get_identity_address(identity):
    url = "%s/%s/%s/addresses/msisdn" % (settings.IDENTITY_STORE_URL,
                                         "identities", identity)
    params = {"default": True}
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, params=params, headers=headers).json()
    if len(r["results"]) > 0:
        return r["results"][0]["address"]
    else:
        return None

def search_identities(params):
    """ Returns the identities matching the given parameters
    """
    url = "%s/%s/search/" % (settings.IDENTITY_STORE_URL, "identities")
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, params=params, headers=headers)
    r.raise_for_status()
    r = r.json()

    while True:
        for identity in r.get('results', []):
            yield identity
        if r.get('next'):
            r = session.get(r['next'], headers=headers)
            r.raise_for_status()
            r = r.json()
        else:
            break

def patch_identity(identity, data):
    """ Patches the given identity with the data provided
    """
    url = "%s/%s/%s/" % (settings.IDENTITY_STORE_URL, "identities", identity)
    data = data
    headers = {
        'Authorization': 'Token %s' % settings.IDENTITY_STORE_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.patch(url, data=json.dumps(data), headers=headers)
    r.raise_for_status()
    return r.json()

def get_messageset_by_shortname(short_name):
    url = "%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL, "messageset")
    params = {'short_name': short_name}
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, params=params, headers=headers)
    return r.json()["results"][0]  # messagesets should be unique, return 1st

def get_messageset(messageset_id):
    url = "%s/%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL, "messageset",
                         messageset_id)
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, headers=headers)
    return r.json()

def get_schedule(schedule_id):
    url = "%s/%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL,
                         "schedule", schedule_id)
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, headers=headers)
    return r.json()

def get_subscriptions(identity):
    """ Gets the active subscriptions for an identity
    """
    url = "%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL, "subscriptions")
    params = {'id': identity, 'active': True}
    headers = {
        'Authorization': 'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.get(url, params=params, headers=headers)
    return r.json()["results"]

def patch_subscription(subscription, data):
    """ Patches the given subscription with the data provided
    """
    url = "%s/%s/%s/" % (settings.STAGE_BASED_MESSAGING_URL,
                         "subscriptions", subscription["id"])
    data = data
    headers = {
        'Authorization':
        'Token %s' % settings.STAGE_BASED_MESSAGING_TOKEN,
        'Content-Type': 'application/json'
    }
    r = session.patch(url, data=data, headers=headers)
    return r.json()

def deactivate_subscription(subscription):
    """ Sets a subscription deactive via a Patch request
    """
    return patch_subscription(subscription, {"active": False})

def get_messageset_short_name(stage, recipient, msg_type, weeks, voice_days,
                              voice_times):

    if recipient == "household":
        msg_type = "audio"

    if stage == "prebirth":
        week_range = "10_42"
    elif stage == "miscarriage":
        week_range = "0_2"
    elif stage == "postbirth":
        if recipient == "household":
            week_range = "0_52"
        elif 0 <= weeks <= 12:
            week_range = "0_12"
        elif 13 <= weeks <= 52:
            week_range = "13_52"

    if msg_type == "text":
        short_name = "%s.%s.%s.%s" % (
            stage, recipient, msg_type, week_range)
    else:
        short_name = "%s.%s.%s.%s.%s.%s" % (
            stage, recipient, msg_type, week_range, voice_days, voice_times)

    return short_name

def get_messageset_schedule_sequence(short_name, weeks):
    # get messageset
    messageset = get_messageset_by_shortname(short_name)

    messageset_id = messageset["id"]
    schedule_id = messageset["default_schedule"]
    # get schedule
    schedule = get_schedule(schedule_id)

    # calculate next_sequence_number
    # get schedule days of week: comma-seperated str e.g. '1,3' for Mon & Wed
    days_of_week = schedule["day_of_week"]
    # determine how many times a week messages are sent e.g. 2 for '1,3'
    msgs_per_week = len(days_of_week.split(','))
    # determine starting message
    # check if in prebirth stage - only starting messaging in week 10
    if 'miscarriage' in short_name:
        next_sequence_number = 1  # always start loss messages at 1
    elif 'prebirth' in short_name:
        next_sequence_number = msgs_per_week * (
            weeks - settings.PREBIRTH_MIN_WEEKS)
        if next_sequence_number == 0:
            next_sequence_number = 1  # next_sequence_number cannot be 0
    elif '13_52' in short_name:
        next_sequence_number = msgs_per_week * (weeks - 13)
        if next_sequence_number == 0:
            next_sequence_number = 1  # next_sequence_number cannot be 0
    else:
        next_sequence_number = msgs_per_week * weeks
        if next_sequence_number == 0:
            next_sequence_number = 1  # next_sequence_number cannot be 0

    return (messageset_id, schedule_id, next_sequence_number)

def post_message(payload):
    result = session.post(
        url="%s/outbound/" % settings.MESSAGE_SENDER_URL,
        data=json.dumps(payload),
        headers={
            'Content-Type': 'application/json',
            'Authorization': 'Token %s' % (
                settings.MESSAGE_SENDER_TOKEN,)
        }
    ).json()
    return result

def get_available_metrics():
    available_metrics = []
    available_metrics.extend(settings.METRICS_REALTIME)
    available_metrics.extend(settings.METRICS_SCHEDULED)

    sources = Source.objects.all()
    for source in sources:
        # only append usernames with characters that are all alphanumeric
        # and/or underscores
        if re.match(r'\w+$', source.user.username):
            available_metrics.append(
                "registrations.source.%s.sum" % source.user.username)

    return available_metrics

def normalise_string(string):
    """ Strips trailing whitespace from string, lowercases it and replaces
        spaces with underscores
    """
    string = (string.strip()).lower()
    return re.sub(r'\W+', '_', string)

def timestamp_to_epoch(timestamp):
    """
    Takes a timestamp and returns a float representing the unix epoch time.
    """
    return (timestamp - datetime.datetime.utcfromtimestamp(0)).total_seconds()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

def find_min_max_naive(a_ls):
    """Find mix & max in a list by naive algorithm.

    - Time complexity: O(n).
    - Space complexity: O(1).
    """
    _min = a_ls[0]
    _max = a_ls[0]
    for i in range(1, len(a_ls)):
        if a_ls[i] < _min:
            _min = a_ls[i]
        if a_ls[i] > _max:
            _max = a_ls[i]
    return [_min, _max]

def find_min_max_dc(a_ls):
    """Find mix & max in a list by divide and conquer algorithm.

    - Time complexity: O(n).
    - Space complexity: O(1).
    """
    if len(a_ls) == 1:
        return [a_ls[0], a_ls[0]]
    elif len(a_ls) == 2:
        if a_ls[0] < a_ls[1]:
            return [a_ls[0], a_ls[1]]
        else:
            return [a_ls[1], a_ls[0]]
    else:
        len_a = len(a_ls)
        [_min1, _max1] = find_min_max_dc(a_ls[:len_a//2])
        [_min2, _max2] = find_min_max_dc(a_ls[len_a//2:])

        if _min1 < _min2:
            _min = _min1
        else:
            _min = _min2

        if _max1 > _max2:
            _max = _max1
        else:
            _max = _max2

        return [_min, _max]

def find_min_max_seq(a_ls):
    """Find mix & max in a list by sequential algorithm.

    - Time complexity: O(n).
    - Space complexity: O(1).
    """
    if a_ls[0] < a_ls[1]:
        cur_min, cur_max = a_ls[0], a_ls[1]
    else:
        cur_min, cur_max = a_ls[1], a_ls[0]

    for i in range(2, len(a_ls), 2):
        if i + 1 < len(a_ls):
            if a_ls[i] < a_ls[i + 1]:
                _min, _max = a_ls[i], a_ls[i + 1]
            else:
                _min, _max = a_ls[i + 1], a_ls[i]
        else:
            _min, _max = a_ls[i], a_ls[i]

        if _min < cur_min:
            cur_min = _min
        if _max > cur_max:
            cur_max = _max

    return [cur_min, cur_max]

def main():
    import time

    # List with even numbers.
    # a_ls = [0, 1, 2, 3, 4, 5, 6, 7]
    # List with odd numbers.
    # a_ls = [0, 1, 2, 3, 4, 5, 6, 7, 8]
    a_ls = range(int(1e6))
    print('Find min & max for list: {}'.format(a_ls))

    start_time = time.time()
    print('By naive algorithm: {}'
          .format(find_min_max_naive(a_ls)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By divide and conquer algorithm: {}'
          .format(find_min_max_dc(a_ls)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By sequential algorithm: {}'
          .format(find_min_max_seq(a_ls)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<commit_msg>Rename functions & clarify slicing list<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

def minmax_naive(a_ls):
    """Find mix & max in a list by naive algorithm.

    - Time complexity: O(n).
    - Space complexity: O(1).
    """
    _min = a_ls[0]
    _max = a_ls[0]
    for i in range(1, len(a_ls)):
        if a_ls[i] < _min:
            _min = a_ls[i]
        if a_ls[i] > _max:
            _max = a_ls[i]
    return [_min, _max]

def minmax_dc(a_ls):
    """Find mix & max in a list by divide and conquer algorithm.

    - Time complexity: O(n).
    - Space complexity: O(1).
    """
    if len(a_ls) == 1:
        return [a_ls[0], a_ls[0]]
    elif len(a_ls) == 2:
        if a_ls[0] < a_ls[1]:
            return [a_ls[0], a_ls[1]]
        else:
            return [a_ls[1], a_ls[0]]
    else:
        len_a = len(a_ls)
        [_min1, _max1] = minmax_dc(a_ls[:(len_a // 2)])
        [_min2, _max2] = minmax_dc(a_ls[(len_a // 2):])

        if _min1 < _min2:
            _min = _min1
        else:
            _min = _min2

        if _max1 > _max2:
            _max = _max1
        else:
            _max = _max2

        return [_min, _max]

def minmax_seq(a_ls):
    """Find mix & max in a list by sequential algorithm.

    - Time complexity: O(n).
    - Space complexity: O(1).
    """
    if a_ls[0] < a_ls[1]:
        cur_min, cur_max = a_ls[0], a_ls[1]
    else:
        cur_min, cur_max = a_ls[1], a_ls[0]

    for i in range(2, len(a_ls), 2):
        if i + 1 < len(a_ls):
            if a_ls[i] < a_ls[i + 1]:
                _min, _max = a_ls[i], a_ls[i + 1]
            else:
                _min, _max = a_ls[i + 1], a_ls[i]
        else:
            _min, _max = a_ls[i], a_ls[i]

        if _min < cur_min:
            cur_min = _min
        if _max > cur_max:
            cur_max = _max

    return [cur_min, cur_max]

def main():
    import time

    a_ls = range(int(1e6))

    start_time = time.time()
    print('By naive algorithm: {}'
          .format(minmax_naive(a_ls)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By divide and conquer algorithm: {}'
          .format(minmax_dc(a_ls)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By sequential algorithm: {}'
          .format(minmax_seq(a_ls)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from contextlib import contextmanager
from functools import update_wrapper
import atexit
import os

import numpy as np

@contextmanager
def ignore_invalid():
    err = np.seterr(invalid='ignore')
    try:
        yield
    finally:
        np.seterr(**err)

def check_array_like(a, *ndims, **kwargs):
    if not hasattr(a, 'ndim'):
        cls = kwargs.pop('default', np.asarray)
        a = cls(a, **kwargs)
    if a.ndim not in ndims:
        raise ValueError('invalid number of dimensions: %s' % a.ndim)

def asarray_ndim(a, *ndims, **kwargs):
    """Ensure numpy array.

    Parameters
    ----------
    a : array_like
    *ndims : int, optional
        Allowed values for number of dimensions.
    **kwargs
        Passed through to :func:`numpy.array`.

    Returns
    -------
    a : numpy.ndarray

    """
    allow_none = kwargs.pop('allow_none', False)
    kwargs.setdefault('copy', False)
    if a is None and allow_none:
        return None
    a = np.array(a, **kwargs)
    if a.ndim not in ndims:
        if len(ndims) > 1:
            expect_str = 'one of %s' % str(ndims)
        else:
            # noinspection PyUnresolvedReferences
            expect_str = '%s' % ndims[0]
        raise TypeError('bad number of dimensions: expected %s; found %s' %
                        (expect_str, a.ndim))
    return a

def check_ndim(a, ndim):
    if a.ndim != ndim:
        raise TypeError('bad number of dimensions: expected %s; found %s' % (ndim, a.ndim))

def check_shape(a, shape):
    if a.shape != shape:
        raise TypeError('bad shape: expected %s; found %s' % (shape, a.shape))

def check_dtype(a, *dtypes):
    dtypes = [np.dtype(t) for t in dtypes]
    if a.dtype not in dtypes:
        raise TypeError('bad dtype: expected on of %s; found %s' % (dtypes, a.dtype))

def check_dtype_kind(a, *kinds):
    if a.dtype.kind not in kinds:
        raise TypeError('bad dtype kind: expected on of %s; found %s' % (kinds, a.dtype.kind))

def check_integer_dtype(a):
    check_dtype_kind(a, 'u', 'i')

def check_dim0_aligned(*arrays):
    check_dim_aligned(0, *arrays)

def check_dim1_aligned(*arrays):
    check_dim_aligned(1, *arrays)

def check_dim_aligned(dim, *arrays):
    a = arrays[0]
    for b in arrays[1:]:
        if b.shape[dim] != a.shape[dim]:
            raise ValueError(
                'arrays do not have matching length for dimension %s' % dim
            )

def check_same_ndim(*arrays):
    a = arrays[0]
    for b in arrays[1:]:
        if len(b.shape) != len(a.shape):
            raise ValueError(
                'arrays do not have same number of dimensions'
            )

def check_equal_length(a, *others):
    expected_length = len(a)
    for b in others:
        if len(b) != expected_length:
            raise ValueError('sequences do not have matching length')

def resize_dim1(a, l, fill=0):
    if a.shape[1] < l:
        newshape = a.shape[0], l
        b = np.zeros(newshape, dtype=a.dtype)
        if fill != 0:
            b.fill(fill)
        b[:, :a.shape[1]] = a
        return b
    else:
        return a

def ensure_dim1_aligned(*arrays, **kwargs):
    fill = kwargs.get('fill', 0)
    dim1_length = max(a.shape[1] for a in arrays)
    arrays = [resize_dim1(a, dim1_length, fill=fill) for a in arrays]
    return arrays

def ensure_square(dist):
    from scipy.spatial.distance import squareform
    dist = asarray_ndim(dist, 1, 2)
    if dist.ndim == 1:
        dist = squareform(dist)
    else:
        if dist.shape[0] != dist.shape[1]:
            raise ValueError('distance matrix is not square')
    return dist

class _HashedSeq(list):

    __slots__ = 'hashvalue'

    # noinspection PyShadowingBuiltins,PyMissingConstructor
    def __init__(self, tup, hash=hash):
        self[:] = tup
        self.hashvalue = hash(tup)

    def __hash__(self):
        return self.hashvalue

# noinspection PyShadowingBuiltins
def _make_key(args, kwds, typed,
              kwd_mark=('__kwargs__',),
              fasttypes=(int, str, frozenset, type(None)),
              sorted=sorted, tuple=tuple, type=type, len=len):
    key = args
    kwd_items = sorted(kwds.items())
    if kwds:
        key += kwd_mark
        for item in kwd_items:
            key += item
    if typed:
        key += tuple(type(v) for v in args)
        if kwds:
            key += tuple(type(v) for _, v in kwd_items)
    else:
        key = args
    if len(key) == 1 and type(key[0]) in fasttypes:
        return key[0]
    return _HashedSeq(key)

def _hdf5_cache_act(filepath, parent, container, key, names, no_cache,
                    user_function, args, kwargs, h5dcreate_kwargs):
    import h5py

    # open the HDF5 file
    with h5py.File(filepath, mode='a') as h5f:

        # find parent group
        if parent is None:
            # use root group
            h5g_parent = h5f
        else:
            h5g_parent = h5f.require_group(parent)

        # find cache container group
        h5g_container = h5g_parent.require_group(container)

        # find cache group
        h5g = h5g_container.require_group(key)

        # call user function and (re)build cache
        if no_cache or '__success__' not in h5g.attrs:

            # reset success mark if present
            if '__success__' in h5g.attrs:
                del h5g.attrs['__success__']

            # compute result
            result = user_function(*args, **kwargs)

            # handle tuple of return values
            if isinstance(result, tuple):

                # determine dataset names
                if names is None:
                    names = ['f%02d' % i for i in range(len(result))]
                elif len(names) < len(result):
                    names = list(names) + ['f%02d' % i
                                           for i in range(len(names),
                                                          len(result))]

                # save data
                for n, r in zip(names, result):
                    if n in h5g:
                        del h5g[n]
                    if np.isscalar(r):
                        h5g.create_dataset(n, data=r)
                    else:
                        h5g.create_dataset(n, data=r, **h5dcreate_kwargs)

            # handle single return value
            else:

                # determine dataset name
                if names is None:
                    n = 'data'
                elif isinstance(names, str):
                    n = names
                elif len(names) > 0:
                    n = names[0]
                else:
                    n = 'data'

                # save data
                if n in h5g:
                    del h5g[n]
                if np.isscalar(result):
                    h5g.create_dataset(n, data=result)
                else:
                    h5g.create_dataset(n, data=result,
                                       **h5dcreate_kwargs)

            # mark success
            h5g.attrs['__success__'] = True

        # load from cache
        else:

            # determine dataset names
            if names is None:
                names = sorted(h5g.keys())
            elif isinstance(names, str):
                names = (names,)

            # load result from cache
            if len(names) == 1:
                result = h5g[names[0]]
                result = result[:] if len(result.shape) > 0 else result[()]
            else:
                result = tuple(h5g[n] for n in names)
                result = tuple(r[:] if len(r.shape) > 0 else r[()]
                               for r in result)

        return result

def hdf5_cache(filepath=None, parent=None, group=None, names=None, typed=False,
               hashed_key=False, **h5dcreate_kwargs):
    """HDF5 cache decorator.

    Parameters
    ----------
    filepath : string, optional
        Path to HDF5 file. If None a temporary file name will be used.
    parent : string, optional
        Path to group within HDF5 file to use as parent. If None the root
        group will be used.
    group : string, optional
        Path to group within HDF5 file, relative to parent, to use as
        container for cached data. If None the name of the wrapped function
        will be used.
    names : sequence of strings, optional
        Name(s) of dataset(s). If None, default names will be 'f00', 'f01',
        etc.
    typed : bool, optional
        If True, arguments of different types will be cached separately.
        For example, f(3.0) and f(3) will be treated as distinct calls with
        distinct results.
    hashed_key : bool, optional
        If False (default) the key will not be hashed, which makes for
        readable cache group names. If True the key will be hashed, however
        note that on Python >= 3.3 the hash value will not be the same between
        sessions unless the environment variable PYTHONHASHSEED has been set
        to the same value.

    Returns
    -------
    decorator : function

    Examples
    --------

    Without any arguments, will cache using a temporary HDF5 file::

        >>> import allel
        >>> @allel.util.hdf5_cache()
        ... def foo(n):
        ...     print('executing foo')
        ...     return np.arange(n)
        ...
        >>> foo(3)
        executing foo
        array([0, 1, 2])
        >>> foo(3)
        array([0, 1, 2])
        >>> foo.cache_filepath # doctest: +SKIP
        '/tmp/tmp_jwtwgjz'

    Supports multiple return values, including scalars, e.g.::

        >>> @allel.util.hdf5_cache()
        ... def bar(n):
        ...     print('executing bar')
        ...     a = np.arange(n)
        ...     return a, a**2, n**2
        ...
        >>> bar(3)
        executing bar
        (array([0, 1, 2]), array([0, 1, 4]), 9)
        >>> bar(3)
        (array([0, 1, 2]), array([0, 1, 4]), 9)

    Names can also be specified for the datasets, e.g.::

        >>> @allel.util.hdf5_cache(names=['z', 'x', 'y'])
        ... def baz(n):
        ...     print('executing baz')
        ...     a = np.arange(n)
        ...     return a, a**2, n**2
        ...
        >>> baz(3)
        executing baz
        (array([0, 1, 2]), array([0, 1, 4]), 9)
        >>> baz(3)
        (array([0, 1, 2]), array([0, 1, 4]), 9)

    """

    # initialise HDF5 file path
    if filepath is None:
        import tempfile
        filepath = tempfile.mktemp(prefix='scikit_allel_', suffix='.h5')
        atexit.register(os.remove, filepath)

    # initialise defaults for dataset creation
    h5dcreate_kwargs.setdefault('chunks', True)

    def decorator(user_function):

        # setup the name for the cache container group
        if group is None:
            container = user_function.__name__
        else:
            container = group

        def wrapper(*args, **kwargs):

            # load from cache or not
            no_cache = kwargs.pop('no_cache', False)

            # compute a key from the function arguments
            key = _make_key(args, kwargs, typed)
            if hashed_key:
                key = str(hash(key))
            else:
                key = str(key).replace('/', '__slash__')

            return _hdf5_cache_act(filepath, parent, container, key, names,
                                   no_cache, user_function, args, kwargs,
                                   h5dcreate_kwargs)

        wrapper.cache_filepath = filepath
        return update_wrapper(wrapper, user_function)

    return decorator

def contains_newaxis(item):
    if item is None:
        return True
    elif item is np.newaxis:
        return True
    elif isinstance(item, tuple):
        return any((i is None or i is np.newaxis) for i in item)
    return False

def check_ploidy(actual, expect):
    if expect != actual:
        raise ValueError(
            'expected ploidy %s, found %s' % (expect, actual)
        )

def check_min_samples(actual, expect):
    if actual < expect:
        raise ValueError(
            'expected at least %s samples, found %s' % (expect, actual)
        )

def check_type(obj, expected):
    if not isinstance(obj, expected):
        raise TypeError('bad argument type, expected %s, found %s' % (expected, type(obj)))
<commit_msg>l can't be var name<commit_after># -*- coding: utf-8 -*-
from contextlib import contextmanager
from functools import update_wrapper
import atexit
import os

import numpy as np

@contextmanager
def ignore_invalid():
    err = np.seterr(invalid='ignore')
    try:
        yield
    finally:
        np.seterr(**err)

def check_array_like(a, *ndims, **kwargs):
    if not hasattr(a, 'ndim'):
        cls = kwargs.pop('default', np.asarray)
        a = cls(a, **kwargs)
    if a.ndim not in ndims:
        raise ValueError('invalid number of dimensions: %s' % a.ndim)

def asarray_ndim(a, *ndims, **kwargs):
    """Ensure numpy array.

    Parameters
    ----------
    a : array_like
    *ndims : int, optional
        Allowed values for number of dimensions.
    **kwargs
        Passed through to :func:`numpy.array`.

    Returns
    -------
    a : numpy.ndarray

    """
    allow_none = kwargs.pop('allow_none', False)
    kwargs.setdefault('copy', False)
    if a is None and allow_none:
        return None
    a = np.array(a, **kwargs)
    if a.ndim not in ndims:
        if len(ndims) > 1:
            expect_str = 'one of %s' % str(ndims)
        else:
            # noinspection PyUnresolvedReferences
            expect_str = '%s' % ndims[0]
        raise TypeError('bad number of dimensions: expected %s; found %s' %
                        (expect_str, a.ndim))
    return a

def check_ndim(a, ndim):
    if a.ndim != ndim:
        raise TypeError('bad number of dimensions: expected %s; found %s' % (ndim, a.ndim))

def check_shape(a, shape):
    if a.shape != shape:
        raise TypeError('bad shape: expected %s; found %s' % (shape, a.shape))

def check_dtype(a, *dtypes):
    dtypes = [np.dtype(t) for t in dtypes]
    if a.dtype not in dtypes:
        raise TypeError('bad dtype: expected on of %s; found %s' % (dtypes, a.dtype))

def check_dtype_kind(a, *kinds):
    if a.dtype.kind not in kinds:
        raise TypeError('bad dtype kind: expected on of %s; found %s' % (kinds, a.dtype.kind))

def check_integer_dtype(a):
    check_dtype_kind(a, 'u', 'i')

def check_dim0_aligned(*arrays):
    check_dim_aligned(0, *arrays)

def check_dim1_aligned(*arrays):
    check_dim_aligned(1, *arrays)

def check_dim_aligned(dim, *arrays):
    a = arrays[0]
    for b in arrays[1:]:
        if b.shape[dim] != a.shape[dim]:
            raise ValueError(
                'arrays do not have matching length for dimension %s' % dim
            )

def check_same_ndim(*arrays):
    a = arrays[0]
    for b in arrays[1:]:
        if len(b.shape) != len(a.shape):
            raise ValueError(
                'arrays do not have same number of dimensions'
            )

def check_equal_length(a, *others):
    expected_length = len(a)
    for b in others:
        if len(b) != expected_length:
            raise ValueError('sequences do not have matching length')

def resize_dim1(a, s, fill=0):
    if a.shape[1] < s:
        newshape = a.shape[0], s
        b = np.zeros(newshape, dtype=a.dtype)
        if fill != 0:
            b.fill(fill)
        b[:, :a.shape[1]] = a
        return b
    else:
        return a

def ensure_dim1_aligned(*arrays, **kwargs):
    fill = kwargs.get('fill', 0)
    dim1_length = max(a.shape[1] for a in arrays)
    arrays = [resize_dim1(a, dim1_length, fill=fill) for a in arrays]
    return arrays

def ensure_square(dist):
    from scipy.spatial.distance import squareform
    dist = asarray_ndim(dist, 1, 2)
    if dist.ndim == 1:
        dist = squareform(dist)
    else:
        if dist.shape[0] != dist.shape[1]:
            raise ValueError('distance matrix is not square')
    return dist

class _HashedSeq(list):

    __slots__ = 'hashvalue'

    # noinspection PyShadowingBuiltins,PyMissingConstructor
    def __init__(self, tup, hash=hash):
        self[:] = tup
        self.hashvalue = hash(tup)

    def __hash__(self):
        return self.hashvalue

# noinspection PyShadowingBuiltins
def _make_key(args, kwds, typed,
              kwd_mark=('__kwargs__',),
              fasttypes=(int, str, frozenset, type(None)),
              sorted=sorted, tuple=tuple, type=type, len=len):
    key = args
    kwd_items = sorted(kwds.items())
    if kwds:
        key += kwd_mark
        for item in kwd_items:
            key += item
    if typed:
        key += tuple(type(v) for v in args)
        if kwds:
            key += tuple(type(v) for _, v in kwd_items)
    else:
        key = args
    if len(key) == 1 and type(key[0]) in fasttypes:
        return key[0]
    return _HashedSeq(key)

def _hdf5_cache_act(filepath, parent, container, key, names, no_cache,
                    user_function, args, kwargs, h5dcreate_kwargs):
    import h5py

    # open the HDF5 file
    with h5py.File(filepath, mode='a') as h5f:

        # find parent group
        if parent is None:
            # use root group
            h5g_parent = h5f
        else:
            h5g_parent = h5f.require_group(parent)

        # find cache container group
        h5g_container = h5g_parent.require_group(container)

        # find cache group
        h5g = h5g_container.require_group(key)

        # call user function and (re)build cache
        if no_cache or '__success__' not in h5g.attrs:

            # reset success mark if present
            if '__success__' in h5g.attrs:
                del h5g.attrs['__success__']

            # compute result
            result = user_function(*args, **kwargs)

            # handle tuple of return values
            if isinstance(result, tuple):

                # determine dataset names
                if names is None:
                    names = ['f%02d' % i for i in range(len(result))]
                elif len(names) < len(result):
                    names = list(names) + ['f%02d' % i
                                           for i in range(len(names),
                                                          len(result))]

                # save data
                for n, r in zip(names, result):
                    if n in h5g:
                        del h5g[n]
                    if np.isscalar(r):
                        h5g.create_dataset(n, data=r)
                    else:
                        h5g.create_dataset(n, data=r, **h5dcreate_kwargs)

            # handle single return value
            else:

                # determine dataset name
                if names is None:
                    n = 'data'
                elif isinstance(names, str):
                    n = names
                elif len(names) > 0:
                    n = names[0]
                else:
                    n = 'data'

                # save data
                if n in h5g:
                    del h5g[n]
                if np.isscalar(result):
                    h5g.create_dataset(n, data=result)
                else:
                    h5g.create_dataset(n, data=result,
                                       **h5dcreate_kwargs)

            # mark success
            h5g.attrs['__success__'] = True

        # load from cache
        else:

            # determine dataset names
            if names is None:
                names = sorted(h5g.keys())
            elif isinstance(names, str):
                names = (names,)

            # load result from cache
            if len(names) == 1:
                result = h5g[names[0]]
                result = result[:] if len(result.shape) > 0 else result[()]
            else:
                result = tuple(h5g[n] for n in names)
                result = tuple(r[:] if len(r.shape) > 0 else r[()]
                               for r in result)

        return result

def hdf5_cache(filepath=None, parent=None, group=None, names=None, typed=False,
               hashed_key=False, **h5dcreate_kwargs):
    """HDF5 cache decorator.

    Parameters
    ----------
    filepath : string, optional
        Path to HDF5 file. If None a temporary file name will be used.
    parent : string, optional
        Path to group within HDF5 file to use as parent. If None the root
        group will be used.
    group : string, optional
        Path to group within HDF5 file, relative to parent, to use as
        container for cached data. If None the name of the wrapped function
        will be used.
    names : sequence of strings, optional
        Name(s) of dataset(s). If None, default names will be 'f00', 'f01',
        etc.
    typed : bool, optional
        If True, arguments of different types will be cached separately.
        For example, f(3.0) and f(3) will be treated as distinct calls with
        distinct results.
    hashed_key : bool, optional
        If False (default) the key will not be hashed, which makes for
        readable cache group names. If True the key will be hashed, however
        note that on Python >= 3.3 the hash value will not be the same between
        sessions unless the environment variable PYTHONHASHSEED has been set
        to the same value.

    Returns
    -------
    decorator : function

    Examples
    --------

    Without any arguments, will cache using a temporary HDF5 file::

        >>> import allel
        >>> @allel.util.hdf5_cache()
        ... def foo(n):
        ...     print('executing foo')
        ...     return np.arange(n)
        ...
        >>> foo(3)
        executing foo
        array([0, 1, 2])
        >>> foo(3)
        array([0, 1, 2])
        >>> foo.cache_filepath # doctest: +SKIP
        '/tmp/tmp_jwtwgjz'

    Supports multiple return values, including scalars, e.g.::

        >>> @allel.util.hdf5_cache()
        ... def bar(n):
        ...     print('executing bar')
        ...     a = np.arange(n)
        ...     return a, a**2, n**2
        ...
        >>> bar(3)
        executing bar
        (array([0, 1, 2]), array([0, 1, 4]), 9)
        >>> bar(3)
        (array([0, 1, 2]), array([0, 1, 4]), 9)

    Names can also be specified for the datasets, e.g.::

        >>> @allel.util.hdf5_cache(names=['z', 'x', 'y'])
        ... def baz(n):
        ...     print('executing baz')
        ...     a = np.arange(n)
        ...     return a, a**2, n**2
        ...
        >>> baz(3)
        executing baz
        (array([0, 1, 2]), array([0, 1, 4]), 9)
        >>> baz(3)
        (array([0, 1, 2]), array([0, 1, 4]), 9)

    """

    # initialise HDF5 file path
    if filepath is None:
        import tempfile
        filepath = tempfile.mktemp(prefix='scikit_allel_', suffix='.h5')
        atexit.register(os.remove, filepath)

    # initialise defaults for dataset creation
    h5dcreate_kwargs.setdefault('chunks', True)

    def decorator(user_function):

        # setup the name for the cache container group
        if group is None:
            container = user_function.__name__
        else:
            container = group

        def wrapper(*args, **kwargs):

            # load from cache or not
            no_cache = kwargs.pop('no_cache', False)

            # compute a key from the function arguments
            key = _make_key(args, kwargs, typed)
            if hashed_key:
                key = str(hash(key))
            else:
                key = str(key).replace('/', '__slash__')

            return _hdf5_cache_act(filepath, parent, container, key, names,
                                   no_cache, user_function, args, kwargs,
                                   h5dcreate_kwargs)

        wrapper.cache_filepath = filepath
        return update_wrapper(wrapper, user_function)

    return decorator

def contains_newaxis(item):
    if item is None:
        return True
    elif item is np.newaxis:
        return True
    elif isinstance(item, tuple):
        return any((i is None or i is np.newaxis) for i in item)
    return False

def check_ploidy(actual, expect):
    if expect != actual:
        raise ValueError(
            'expected ploidy %s, found %s' % (expect, actual)
        )

def check_min_samples(actual, expect):
    if actual < expect:
        raise ValueError(
            'expected at least %s samples, found %s' % (expect, actual)
        )

def check_type(obj, expected):
    if not isinstance(obj, expected):
        raise TypeError('bad argument type, expected %s, found %s' % (expected, type(obj)))
<|endoftext|>


<commit_before>"""usage: purpletag score [options]

Compute polarity scores for all .tags files that we haven't yet processed.

Options
    -h, --help             help
    -r, --refresh-mocs     fetch latest legislator information from GovTrack
    -c, --counts           use hashtag count features instead of binary features
    -o, --overwrite        overwrite existing .scores files
"""
from docopt import docopt
import io

import numpy as np
from scipy.sparse import csr_matrix
from scipy.stats.mstats import zscore
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_selection import chi2
from sklearn.preprocessing import LabelEncoder

from . import config
from data import get_basenames, get_files, fetch_legislators, twitter_handle_to_party

def has_score_file(tag_file, score_files_bn):
    """ Return true if tag_file has a corresponding score_file. """
    bn = get_basenames([tag_file])[0]
    return bn in score_files_bn

def parse_tags(parts, args):
    """ Parse a line of a .tags file into a dict from tag to count.
    >>> sorted(parse_tags(['foo:10', 'bar:5'], {'--counts':True}).items())
    [('bar', 5.0), ('foo', 10.0)]
    >>> sorted(parse_tags(['foo:10', 'bar:5'], {'--counts':False}).items())
    [('bar', 1.0), ('foo', 1.0)]
    """
    tags = {}
    for t in parts:
        tag_val = t.split(':')
        if not args['--counts']:
            tag_val[1] = 1.
        tags[tag_val[0]] = float(tag_val[1])
    return tags

def parse_tag_file(tag_file, handle2party, args):
    """ Return list of twitter handles, tag counts, and labels. """
    inf = io.open(tag_file, mode='rt', encoding='utf8')
    labels = []
    dicts = []
    handles = []
    for line in inf:
        parts = line.strip().split()
        handle = parts[0]
        if handle in handle2party:
            labels.append(handle2party[handle])
            dicts.append(parse_tags(parts[1:], args))
            handles.append(handle)
    return (handles, dicts, labels)

def feat_counts(fi, ci, X, y):
    """
    Count the number of times feature fi occurs in a row with class ci.

    >>> feat_counts(0, 0, csr_matrix([[0], [1], [0], [1]]), [0, 1, 0, 0])
    1
    >>> feat_counts(0, 1, csr_matrix([[0], [1], [0], [1]]), [0, 1, 0, 1])
    2
    """
    indices = [idx for idx, val in enumerate(y) if val == ci]
    return X[indices][:, fi].sum()

def feat_class_pr(X, y, ci, V):
   return (1. + np.array([feat_counts(i, ci, X, y) for i in range(V)])) \
       / (V + len([yi for yi in y if yi == ci]))

def compute_signs(X, y, V):
    """ For each feature, return 1 if correlated with class 1, else return -1.
    >>> compute_signs(csr_matrix([[1, 0], [1, 0], [0, 1], [0, 1]]), [1, 1, 0, 0], 2)
    [1.0, -1.0]
    """
    pr0 = feat_class_pr(X, y, 0, V)
    pr1 = feat_class_pr(X, y, 1, V)
    return [1. if p1 > p0 else -1. for p0, p1 in zip(pr0, pr1)]

def score_features(X, y, V, args):
    """ FIXME: allow different options. """
    chis, pvals = chi2(X, y)
    # chis /= sum(chis)
    signs = compute_signs(X, y, V)
    signs = np.multiply(chis, signs)
    return signs
    # return zscore(signs)

def write_scores(scores, vocab, tag_file):
    basename = get_basenames([tag_file])[0]
    fp = io.open(config.get('data',
                            'path') + '/' + config.get('data', 'scores') + '/' + basename + '.scores',
                 mode='wt', encoding='utf8')
    for word, score in sorted(zip(vocab, scores), key=lambda x: x[1]):
        fp.write('%s %g\n' % (word, score))
    fp.close()

def score(tag_file, handle2party, args):
    """ Output a file with hashtag scores. """
    handles, tags, parties = parse_tag_file(tag_file, handle2party, args)
    if len(tags) == 0:
        print "can't find any legislators in %s" % tag_file
        return

    vec = DictVectorizer()
    X = vec.fit_transform(tags)
    print '5 features:', vec.get_feature_names()[:5]
    label_enc = LabelEncoder()
    y = label_enc.fit_transform(parties)
    print 'vectorized %d instances' % len(y)
    print 'classes=', label_enc.classes_
    scores = score_features(X, y, len(vec.vocabulary_), args)
    write_scores(scores, vec.get_feature_names(), tag_file)

def get_tag_files(overwrite):
    """ Get .tags files. """
    tag_files = get_files('tags', 'tags')
    if overwrite:
        return tag_files
    else:
        score_files_bn = get_basenames(get_files('scores', 'scores'))
        return [f for f in tag_files if not has_score_file(f, score_files_bn)]

def main():
    args = docopt(__doc__)
    if args['--refresh-mocs']:
        print 'refreshing MOCs'
        fetch_legislators()
    print 'reading twitter handles'
    handle2party = twitter_handle_to_party()
    print handle2party.items()[0]
    tag_files = get_tag_files(args['--overwrite'])
    for tag_file in tag_files:
        print 'scoring', tag_file
        score(tag_file, handle2party, args)

if __name__ == '__main__':
    main()
<commit_msg>added score_mocs function<commit_after>"""usage: purpletag score [options]

Compute polarity scores for all .tags files that we haven't yet processed.

Options
    -h, --help             help
    -r, --refresh-mocs     fetch latest legislator information from GovTrack
    -c, --counts           use hashtag count features instead of binary features
    -o, --overwrite        overwrite existing .scores files
    -s, --score-mocs       calculate MOC scores from tag scores
"""
from docopt import docopt
import io

import numpy as np
from scipy.sparse import csr_matrix
from scipy.stats.mstats import zscore
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_selection import chi2
from sklearn.preprocessing import LabelEncoder

from . import config
from data import get_basenames, get_files, fetch_legislators, twitter_handle_to_party

def has_score_file(tag_file, score_files_bn):
    """ Return true if tag_file has a corresponding score_file. """
    bn = get_basenames([tag_file])[0]
    return bn in score_files_bn

def parse_tags(parts, args):
    """ Parse a line of a .tags file into a dict from tag to count.
    >>> sorted(parse_tags(['foo:10', 'bar:5'], {'--counts':True}).items())
    [('bar', 5.0), ('foo', 10.0)]
    >>> sorted(parse_tags(['foo:10', 'bar:5'], {'--counts':False}).items())
    [('bar', 1.0), ('foo', 1.0)]
    """
    tags = {}
    for t in parts:
        tag_val = t.split(':')
        if not args['--counts']:
            tag_val[1] = 1.
        tags[tag_val[0]] = float(tag_val[1])
    return tags

def parse_tag_file(tag_file, handle2party, args):
    """ Return list of twitter handles, tag counts, and labels. """
    inf = io.open(tag_file, mode='rt', encoding='utf8')
    labels = []
    dicts = []
    handles = []
    for line in inf:
        parts = line.strip().split()
        handle = parts[0]
        if handle in handle2party:
            labels.append(handle2party[handle])
            dicts.append(parse_tags(parts[1:], args))
            handles.append(handle)
    return (handles, dicts, labels)

def feat_counts(fi, ci, X, y):
    """
    Count the number of times feature fi occurs in a row with class ci.

    >>> feat_counts(0, 0, csr_matrix([[0], [1], [0], [1]]), [0, 1, 0, 0])
    1
    >>> feat_counts(0, 1, csr_matrix([[0], [1], [0], [1]]), [0, 1, 0, 1])
    2
    """
    indices = [idx for idx, val in enumerate(y) if val == ci]
    return X[indices][:, fi].sum()

def feat_class_pr(X, y, ci, V):
   return (1. + np.array([feat_counts(i, ci, X, y) for i in range(V)])) \
       / (V + len([yi for yi in y if yi == ci]))

def compute_signs(X, y, V):
    """ For each feature, return 1 if correlated with class 1, else return -1.
    >>> compute_signs(csr_matrix([[1, 0], [1, 0], [0, 1], [0, 1]]), [1, 1, 0, 0], 2)
    [1.0, -1.0]
    """
    pr0 = feat_class_pr(X, y, 0, V)
    pr1 = feat_class_pr(X, y, 1, V)
    return [1. if p1 > p0 else -1. for p0, p1 in zip(pr0, pr1)]

def score_features(X, y, V, args):
    """ FIXME: allow different options. """
    chis, pvals = chi2(X, y)
    # chis /= sum(chis)
    signs = compute_signs(X, y, V)
    signs = np.multiply(chis, signs)
    return signs
    # return zscore(signs)

def write_scores(scores, vocab, tag_file):
    basename = get_basenames([tag_file])[0]
    fp = io.open(config.get('data',
                            'path') + '/' + config.get('data', 'scores') + '/' + basename + '.scores',
                 mode='wt', encoding='utf8')
    for word, score in sorted(zip(vocab, scores), key=lambda x: x[1]):
        fp.write('%s %g\n' % (word, score))
    fp.close()

def write_moc_scores(mocs, tag_file):
    basename = get_basenames([tag_file])[0]
    fp = io.open(config.get('data', 'path') + '/' + config.get('data', 'scores') + '/' + basename + '.moc.scores',
                 mode='wt', encoding='utf8')
    for moc, score in mocs.items():
        fp.write('%s %g\n' % (moc, score))
    fp.close()

def score(tag_file, handle2party, args):
    """ Output a file with hashtag scores. """
    handles, tags, parties = parse_tag_file(tag_file, handle2party, args)
    if len(tags) == 0:
        print "can't find any legislators in %s" % tag_file
        return

    vec = DictVectorizer()
    X = vec.fit_transform(tags)
    print '5 features:', vec.get_feature_names()[:5]
    label_enc = LabelEncoder()
    y = label_enc.fit_transform(parties)
    print 'vectorized %d instances' % len(y)
    print 'classes=', label_enc.classes_
    scores = score_features(X, y, len(vec.vocabulary_), args)
    write_scores(scores, vec.get_feature_names(), tag_file)

def get_tag_scores(score_file, args):
    scores = {}
    with open(score_file) as f:
        for line in f:
            (key, val) = line.split()
            scores[key] = val
    return scores

def score_mocs(score_file, tag_file, args):
    """ Output a file with user scores. """
    inf = io.open(tag_file, mode='rt', encoding='utf8')
    scores = get_tag_scores(score_file, args)

    mocs = {}
    # get the MOC's tags
    for line in inf:
        moc_score = 0
        tags = {}
        parts = line.strip().split()
        handle = parts[0]
        tags = parse_tags(parts[1:], args)

        # calculate a MOC score
        for tag in tags:
            if not args['--counts']:
                try:
                    moc_score += float(scores[tag])
                except:
                    print "can't find key for", tag # FIXME: can't handle accented characters in keys
                    continue
            else:
                print "using counts. feature not supported."
        mocs[handle] = moc_score

    # write the score to a file
    write_moc_scores(mocs, tag_file)
    return True

def get_tag_files(overwrite, score_mocs):
    """ Get .tags files. """
    tag_files = get_files('tags', 'tags')
    if overwrite or score_mocs:
        return tag_files
    else:
        score_files_bn = get_basenames(get_files('scores', 'scores'))
        return [f for f in tag_files if not has_score_file(f, score_files_bn)]

def main():
    args = docopt(__doc__)
    if args['--refresh-mocs']:
        print 'refreshing MOCs'
        fetch_legislators()

    # print 'reading twitter handles'
    # handle2party = twitter_handle_to_party()
    # print handle2party.items()[0]
    tag_files = get_tag_files(args['--overwrite'], args['--score-mocs'])
    for tag_file in tag_files:
        # print 'scoring', tag_file
        # score(tag_file, handle2party, args)
        if args['--score-mocs']:
            score_file = tag_file.replace('tags', 'scores')
            print 'scoring MOCs with', score_file
            score_mocs(score_file, tag_file, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""alpha URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf import settings
from django.conf.urls.static import static
from django.conf.urls import include, url
from django.contrib import admin
from django.views.generic import RedirectView

import webapp.urls

urlpatterns = [
    url(r'^$', RedirectView.as_view(url='webapp/')),
    url(r'^admin/', admin.site.urls),
    url(r'^webapp/', include(webapp.urls)), ] + \
    static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<commit_msg>Ordena importaciones<commit_after>"""alpha URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls.static import static
from django.contrib import admin
from django.views.generic import RedirectView

import webapp.urls

urlpatterns = [
    url(r'^$', RedirectView.as_view(url='webapp/')),
    url(r'^admin/', admin.site.urls),
    url(r'^webapp/', include(webapp.urls)), ] + \
    static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<|endoftext|>