<commit_before>#
# Copyright (c) 2004-2005 rpath, Inc.
#
# All Rights Reserved
#

# python standard library imports
import os
import tempfile
import subprocess

# mint imports
from imagegen import ImageGenerator

# conary imports
import conary
import conaryclient
import deps
import flavorcfg
import versions
from callbacks import UpdateCallback, ChangesetCallback
from conarycfg import ConfigFile
from lib import log

class LiveIsoConfig(ConfigFile):
    defaults = {
        'scriptPath':        None,
    }

class InstallCallback(UpdateCallback, ChangesetCallback):
    def restoreFiles(self, size, totalSize):
        if totalSize != 0:
            self.restored += size
            self.update('writing files (%d%% of %dK)'
                        %((self.restored * 100) / totalSize, totalSize / 1024))

    def requestingChangeSet(self):
        self.update('requesting changeset')

    def setUpdateHunk(self, num, total):
        self.restored = 0

    def downloadingChangeSet(self, got, need):
        if need != 0:
            self.update('downloading from repository (%d%% of %dk)' %
                        ((got * 100) / need, need / 1024))

    def update(self, msg):
        # only push an update into the database if it differs from the
        # current message
        if self.msg != msg:
            self.msg = msg
            self.status(msg)

    def __init__(self, status):
        self.status = status
        self.restored = 0
        self.msg = ''

class Journal:
    def lchown(self, root, target, user, group):
        # get rid of the root
        target = target[len(root):]
        dirname = os.path.dirname(target)
        filename = os.path.basename(target)
        f = open(os.sep.join((root, dirname, '.UIDGID')), 'a')
        # XXX e2fsimage does not handle group lookups yet
        f.write('%s %s\n' %(filename, user))
        f.close()

    def mknod(self, root, target, devtype, major, minor, mode,
              uid, gid):
        # get rid of the root
        target = target[len(root):]
        dirname = os.path.dirname(target)
        filename = os.path.basename(target)
        f = open(os.sep.join((root, dirname, '.DEVICES')), 'a')
        # XXX e2fsimage does not handle symbolic users/groups for .DEVICES
        f.write('%s %s %d %d 0%o\n' %(filename, devtype, major, minor, mode))
        f.close()

class LiveIso(ImageGenerator):
    def getConfig(self):
        cfg = LiveIsoConfig()
        cfg.read("live_iso.conf")
        return cfg

    def write(self):
        imgcfg = self.getConfig()
        if not imgcfg.scriptPath:
            raise RuntimeError, 'scriptPath must be set in configuration file'

        tmpDir = tempfile.mkdtemp("", "imagetool", self.cfg.imagesPath)
        log.info('generating live iso with tmpdir %s', tmpDir)
        release = self.client.getRelease(self.job.getReleaseId())
        trove, version, flavorStr = release.getTrove()

        project = self.client.getProject(release.getProjectId())

        trove, versionStr, flavorStr = release.getTrove()
        flavor = deps.deps.ThawDependencySet(flavorStr)
        version = versions.ThawVersion(versionStr)

        project = self.client.getProject(release.getProjectId())

        # set up configuration
        cfg = project.getConaryConfig()
        # turn off threading
        cfg.threadded = False
        flavorConfig = flavorcfg.FlavorConfig(cfg.useDirs, cfg.archDirs)
        cfg.flavor = flavorConfig.toDependency(override=cfg.flavor[0])
        insSet = deps.deps.DependencySet()
        for dep in deps.arch.currentArch:
            insSet.addDep(deps.deps.InstructionSetDependency, dep[0])
        cfg.flavor.union(insSet)
        cfg.buildFlavor = cfg.flavor.copy()
        flavorConfig.populateBuildFlags()
        cfg.setValue('root', tmpDir)

        client = conaryclient.ConaryClient(cfg)
        applyList = [(trove, version, flavor)]
        self.status('installing software')
        callback = InstallCallback(self.status)
        (updJob, suggMap) = client.updateChangeSet(applyList, recurse = True,
                                                   resolveDeps = False,
                                                   callback = callback)
        journal = Journal()
        client.applyUpdate(updJob, journal=journal, callback = callback)

        self.status('generating images')
        fd, fn = tempfile.mkstemp('.iso', 'livecd', self.cfg.imagesPath)
        os.close(fd)

        subprocess.call((os.path.join(imgcfg.scriptPath, 'mklivecd'), tmpDir,
                         fn))
        os.chmod(fn, 0644)

        return [fn]
<commit_msg>remove redundant code<commit_after>#
# Copyright (c) 2004-2005 rpath, Inc.
#
# All Rights Reserved
#

# python standard library imports
import os
import tempfile
import subprocess

# mint imports
from imagegen import ImageGenerator

# conary imports
import conary
import conaryclient
import deps
import flavorcfg
import versions
from callbacks import UpdateCallback, ChangesetCallback
from conarycfg import ConfigFile
from lib import log

class LiveIsoConfig(ConfigFile):
    defaults = {
        'scriptPath':        None,
    }

class InstallCallback(UpdateCallback, ChangesetCallback):
    def restoreFiles(self, size, totalSize):
        if totalSize != 0:
            self.restored += size
            self.update('writing files (%d%% of %dK)'
                        %((self.restored * 100) / totalSize, totalSize / 1024))

    def requestingChangeSet(self):
        self.update('requesting changeset')

    def setUpdateHunk(self, num, total):
        self.restored = 0

    def downloadingChangeSet(self, got, need):
        if need != 0:
            self.update('downloading from repository (%d%% of %dk)' %
                        ((got * 100) / need, need / 1024))

    def update(self, msg):
        # only push an update into the database if it differs from the
        # current message
        if self.msg != msg:
            self.msg = msg
            self.status(msg)

    def __init__(self, status):
        self.status = status
        self.restored = 0
        self.msg = ''

class Journal:
    def lchown(self, root, target, user, group):
        # get rid of the root
        target = target[len(root):]
        dirname = os.path.dirname(target)
        filename = os.path.basename(target)
        f = open(os.sep.join((root, dirname, '.UIDGID')), 'a')
        # XXX e2fsimage does not handle group lookups yet
        f.write('%s %s\n' %(filename, user))
        f.close()

    def mknod(self, root, target, devtype, major, minor, mode,
              uid, gid):
        # get rid of the root
        target = target[len(root):]
        dirname = os.path.dirname(target)
        filename = os.path.basename(target)
        f = open(os.sep.join((root, dirname, '.DEVICES')), 'a')
        # XXX e2fsimage does not handle symbolic users/groups for .DEVICES
        f.write('%s %s %d %d 0%o\n' %(filename, devtype, major, minor, mode))
        f.close()

class LiveIso(ImageGenerator):
    def getConfig(self):
        cfg = LiveIsoConfig()
        cfg.read("live_iso.conf")
        return cfg

    def write(self):
        imgcfg = self.getConfig()
        if not imgcfg.scriptPath:
            raise RuntimeError, 'scriptPath must be set in configuration file'

        tmpDir = tempfile.mkdtemp("", "imagetool", self.cfg.imagesPath)
        log.info('generating live iso with tmpdir %s', tmpDir)

        release = self.client.getRelease(self.job.getReleaseId())
        trove, versionStr, flavorStr = release.getTrove()
        flavor = deps.deps.ThawDependencySet(flavorStr)
        version = versions.ThawVersion(versionStr)

        project = self.client.getProject(release.getProjectId())

        # set up configuration
        cfg = project.getConaryConfig()
        # turn off threading
        cfg.threadded = False
        # configure flavor
        flavorConfig = flavorcfg.FlavorConfig(cfg.useDirs, cfg.archDirs)
        cfg.flavor = flavorConfig.toDependency(override=cfg.flavor[0])
        insSet = deps.deps.DependencySet()
        for dep in deps.arch.currentArch:
            insSet.addDep(deps.deps.InstructionSetDependency, dep[0])
        cfg.flavor.union(insSet)
        cfg.buildFlavor = cfg.flavor.copy()
        flavorConfig.populateBuildFlags()
        cfg.setValue('root', tmpDir)

        client = conaryclient.ConaryClient(cfg)
        applyList = [(trove, version, flavor)]
        self.status('installing software')
        callback = InstallCallback(self.status)
        (updJob, suggMap) = client.updateChangeSet(applyList, recurse = True,
                                                   resolveDeps = False,
                                                   callback = callback)
        journal = Journal()
        client.applyUpdate(updJob, journal=journal, callback = callback)

        self.status('generating images')
        fd, fn = tempfile.mkstemp('.iso', 'livecd', self.cfg.imagesPath)
        os.close(fd)

        subprocess.call((os.path.join(imgcfg.scriptPath, 'mklivecd'), tmpDir,
                         fn))
        os.chmod(fn, 0644)

        return [fn]
<|endoftext|>


<commit_before>import sort
import torch, random, sys
from torch.autograd import Variable
from torch.nn import Parameter
import torch.nn.functional as F
from torch import nn, optim
from tqdm import trange
from tensorboardX import SummaryWriter

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import util, logging, time, gc
import numpy as np
from scipy.stats import sem
from numpy import std

from argparse import ArgumentParser

import torchvision
from torch.utils.data import TensorDataset, DataLoader

import os

from gaussian import HyperLayer

logging.basicConfig(filename='run.log',level=logging.INFO)
LOG = logging.getLogger()

"""
Experiment: learn a mapping from a random x, to x sorted.

"""
tbw = SummaryWriter()

util.DEBUG = False
BUCKET_SIGMA = 0.05

def clean(axes=None):

    if axes is None:
        axes = plt.gca()

    [s.set_visible(False) for s in axes.spines.values()]
    axes.tick_params(top=False, bottom=False, left=False, right=False, labelbottom=False, labelleft=False)

def gen(b, data, size):

    t = torch.zeros(b, size, 1, 28, 28)

    # Select random digits
    for i in range(size):
        sample = random.sample(data[i], k=b)
        t[:, i, :, :, :] = torch.cat(sample, dim=0)

    x = t.clone()

    # Shuffle
    for bi in range(b):
        perm = torch.randperm(size)
        x[bi] = x[bi, perm, :, :, :]

    return x, t

def go(arg):
    """

    :param arg:
    :return:
    """

    """
    Load and organize the data
    """
    trans = torchvision.transforms.ToTensor()
    if arg.final:
        train = torchvision.datasets.MNIST(root=arg.data, train=True, download=True, transform=trans)
        trainloader = torch.utils.data.DataLoader(train, batch_size=arg.batch, shuffle=True, num_workers=2)

        test = torchvision.datasets.MNIST(root=arg.data, train=False, download=True, transform=trans)
        testloader = torch.utils.data.DataLoader(test, batch_size=arg.batch, shuffle=False, num_workers=2)

    else:
        NUM_TRAIN = 45000
        NUM_VAL = 5000
        total = NUM_TRAIN + NUM_VAL

        train = torchvision.datasets.MNIST(root=arg.data, train=True, download=True, transform=trans)

        trainloader = DataLoader(train, batch_size=arg.batch, sampler=util.ChunkSampler(0, NUM_TRAIN, total))
        testloader = DataLoader(train, batch_size=arg.batch, sampler=util.ChunkSampler(NUM_TRAIN, NUM_VAL, total))

    shape = (1, 28, 28)
    num_classes = 10

    train = {label: [] for label in range(10)}

    for inps, labels in trainloader:
        b, c, h, w = inps.size()
        for i in range(b):
            image = inps[i:i+1, :, :, :]
            label = labels[i].item()
            train[label].append(image)

    # train = {label: torch.cat(imgs, dim=0) for label, imgs in train}

    test = {label: [] for label in range(10)}
    for inps, labels in trainloader:
        b, c, h, w = inps.size()
        for i in range(b):
            image = inps[i:i+1, :, :, :]
            label = labels[i].item()
            test[label].append(image)

    # train = {label: torch.cat(imgs, dim=0) for label, imgs in train}
    del b, c, h, w

    torch.manual_seed(arg.seed)

    ndots = arg.iterations // arg.dot_every

    results = np.zeros((arg.reps, ndots))

    for r in range(arg.reps):
        print('starting {} out of {} repetitions'.format(r, arg.reps))
        util.makedirs('./mnist-sort/{}'.format( r))

        model = sort.SortLayer(arg.size, additional=arg.additional, sigma_scale=arg.sigma_scale, sigma_floor=arg.min_sigma)

        # bottom = nn.Linear(28*28, 256)
        # tokeys = nn.Sequential(
        #     util.Flatten(),
        #     bottom, nn.ReLU(),
        #     nn.Linear(256, 64), nn.ReLU(),
        #     nn.Linear(64, 1)
        # )

        # - channel sizes
        c1, c2, c3 = 16, 64, 128
        h1, h2 = 256, 128

        tokeys = nn.Sequential(
            nn.Conv2d(1, c1, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c1, c1, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c1, c1, (3, 3), padding=1), nn.ReLU(),
            nn.BatchNorm2d(c1),
            nn.MaxPool2d((2, 2)),
            nn.Conv2d(c1, c2, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c2, c2, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c2, c2, (3, 3), padding=1), nn.ReLU(),
            nn.BatchNorm2d(c2),
            nn.MaxPool2d((2, 2)),
            nn.Conv2d(c2, c3, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c3, c3, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c3, c3, (3, 3), padding=1), nn.ReLU(),
            nn.BatchNorm2d(c3),
            nn.MaxPool2d((2, 2)),
            util.Flatten(),
            nn.Linear(9 * c3, h1), nn.ReLU(),
            nn.Linear(h1, h2), nn.ReLU(),
            nn.Linear(h2, 1), nn.BatchNorm1d(1),
        )

        if arg.cuda:
            model.cuda()
            tokeys.cuda()

        optimizer = optim.Adam(list(model.parameters()) + list(tokeys.parameters()), lr=arg.lr)

        # seen = 0
        # for ep in trange(50):
        #     for i, (batch, labels) in enumerate(trainloader):
        #
        #         optimizer.zero_grad()
        #
        #         labels = labels.float()
        #         keys = tokeys(batch)
        #
        #         loss = F.mse_loss(keys.squeeze(), labels)
        #
        #         loss.backward()
        #
        #         optimizer.step()
        #
        #         seen += batch.size(0)
        #         tbw.add_scalar('mnist-sort/pretrain-loss', loss.data.item(), seen)

        for i in trange(arg.iterations):

            if i > 3000:
                util.DEBUG = True

            x, t = gen(arg.batch, train, arg.size) # torch.randn((arg.batch_size,) + SHAPE)

            t, idxs = x.sort(dim=1)

            if arg.cuda:
                x, t = x.cuda(), t.cuda()
            x, t = Variable(x), Variable(t)

            optimizer.zero_grad()

            keys = tokeys(x.view(arg.batch * arg.size, 1, 28, 28))
            keys = keys.view(arg.batch, arg.size)

            keys.retain_grad()

            x = x.view(arg.batch, arg.size, -1)
            t = t.view(arg.batch, arg.size, -1)

            y, _ = model(x, keys=keys)

            loss = F.mse_loss(y, t) # compute the loss

            loss.backward()
            # perms = util.DEBUG
            #
            # print(perms)
            # print((keys - keys.mean(dim=1, keepdim=True)).sign())
            # print((keys - keys.mean(dim=1, keepdim=True)).sign() - 0.1 *  keys.grad.sign())
            #
            # k =  (keys - keys.mean(dim=1, keepdim=True)).sign()
            # # print( (perms[:, 0] < perms[:, 1]) == (k[:, 0] < k[:, 1]) )

            optimizer.step()

            tbw.add_scalar('mnist-sort/loss/{}/{}'.format(arg.size, r), loss.data.item(), i*arg.batch)

            # Compute test set loss
            if i % arg.plot_every == 0:
                with torch.no_grad():

                    x, t = gen(arg.batch, test, arg.size)
                    if arg.cuda:
                        x, t = x.cuda(), t.cuda()

                    x, t = Variable(x), Variable(t)

                    keys = tokeys(x.view(arg.batch * arg.size, 1, 28, 28))
                    # keys = tokeys(x.view(arg.batch * arg.size, -1))
                    keys = keys.view(arg.batch, arg.size)

                    x = x.view(arg.batch, arg.size, -1)
                    t = t.view(arg.batch, arg.size, -1)

                    yi, _ = model(x, keys=keys, train=False)
                    yt, _ = model(x, keys=keys, train=True)

                    input  = x[0].view(arg.size, 28, 28).data.cpu().numpy()
                    target = t[0].view(arg.size, 28, 28).data.cpu().numpy()
                    output_inf   = yi[0].view(arg.size, 28, 28).data.cpu().numpy()
                    output_train = yt[0].view(arg.size, 28, 28).data.cpu().numpy()

                    plt.figure(figsize=(arg.size, 4))
                    for col in range(arg.size):
                        ax = plt.subplot(4, arg.size, col + 1)
                        ax.imshow(input[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('input')

                        ax = plt.subplot(4, arg.size, col + arg.size + 1)
                        ax.imshow(target[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('target')

                        ax = plt.subplot(4, arg.size, col + arg.size * 2 + 1)
                        ax.imshow(output_inf[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('inference')

                        ax = plt.subplot(4, arg.size, col + arg.size * 3 + 1)
                        ax.imshow(output_train[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('training')

                    plt.savefig('./mnist-sort/mnist.{:04}.pdf'.format(i))
                    # sys.exit()

            if i % arg.dot_every == 0 and False:
                # print('gradient ', bottom.weight.grad.mean(), bottom.weight.grad.var())

                with torch.no_grad():

                    losses = []
                    for ii in range(10000//arg.batch):
                        x, t = gen(arg.batch, test, arg.size)

                        if arg.cuda:
                            x, t = x.cuda(), t.cuda()

                        x, t = Variable(x), Variable(t)

                        keys = tokeys(x.view(arg.batch * arg.size, 1, 28, 28))
                        #keys = tokeys(x.view(arg.batch * arg.size, -1))
                        keys = keys.view(arg.batch, arg.size)

                        x = x.view(arg.batch, arg.size, -1)
                        t = t.view(arg.batch, arg.size, -1)

                        y, _ = model(x, keys=keys, train=False)

                        loss = F.mse_loss(y, t)  # compute the loss

                        losses.append(loss.item())

                    print('loss', np.mean(losses))

                    results[r, i//arg.dot_every] = np.mean(losses)

                    tbw.add_scalar('mnist-sort/testloss/{}/{}'.format(arg.size, r), np.mean(losses), i * arg.batch)

    np.save('results.{}.np'.format(arg.size), results)
    print('experiments finished')

    plt.figure(figsize=(10, 5))
    ax = plt.gca()

    if results.shape[0] > 1:
        ax.errorbar(x=np.arange(ndots) * arg.dot_every, y=np.mean(results[:, :], axis=0),
                        yerr=np.std(results[:, :], axis=0),
                        label='size {0}x{0}, r={1}'.format(arg.size, arg.reps))
    else:
        ax.plot(np.arange(ndots) * arg.dot_every, np.mean(results[:, :], axis=0),
                        label='size {0}x{0}'.format(arg.size))

    ax.legend()

    util.basic(ax)

    ax.spines['bottom'].set_position('zero')
    ax.set_ylim(0.0, 1.0)
#    ax.set_xlim(0.0, 100.0)

    plt.xlabel('iterations')
    plt.ylabel('error')

    plt.savefig('./quicksort/result.png')
    plt.savefig('./quicksort/result.pdf')

if __name__ == "__main__":

    ## Parse the command line options
    parser = ArgumentParser()

    parser.add_argument("-s", "--size",
                        dest="size",
                        help="Dimensionality of the input.",
                        default=8, type=int)

    parser.add_argument("-b", "--batch-size",
                        dest="batch",
                        help="The batch size.",
                        default=128, type=int)

    parser.add_argument("-i", "--iterations",
                        dest="iterations",
                        help="Number of iterations (in batches).",
                        default=8000, type=int)

    parser.add_argument("-a", "--additional",
                        dest="additional",
                        help="Number of additional points sampled globally",
                        default=2, type=int)

    parser.add_argument("-c", "--cuda", dest="cuda",
                        help="Whether to use cuda.",
                        action="store_true")

    parser.add_argument("-l", "--learn-rate",
                        dest="lr",
                        help="Learning rate",
                        default=0.0001, type=float)

    parser.add_argument("-p", "--plot-every",
                        dest="plot_every",
                        help="Plot every x iterations",
                        default=50, type=int)

    parser.add_argument("-r", "--random-seed",
                        dest="seed",
                        help="Random seed.",
                        default=32, type=int)

    parser.add_argument("-d", "--dot-every",
                        dest="dot_every",
                        help="How many iterations per dot in the loss curves.",
                        default=1000, type=int)

    parser.add_argument("-D", "--data",
                        dest="data",
                        help="Data ditectory.",
                        default='./data', type=str)

    parser.add_argument("-S", "--sigma-scale",
                        dest="sigma_scale",
                        help="Sigma scale.",
                        default=0.1, type=float)

    parser.add_argument("-R", "--repeats",
                        dest="reps",
                        help="Number of repeats.",
                        default=10, type=int)

    parser.add_argument("-M", "--min-sigma",
                        dest="min_sigma",
                        help="Sigma floor (minimum sigma value).",
                        default=0.0, type=float)

    parser.add_argument("-f", "--final", dest="final",
                        help="Whether to run on the real test set.",
                        action="store_true")

    options = parser.parse_args()

    print('OPTIONS ', options)
    LOG.info('OPTIONS ' + str(options))

    go(options)
<commit_msg>limit argument<commit_after>import sort
import torch, random, sys
from torch.autograd import Variable
from torch.nn import Parameter
import torch.nn.functional as F
from torch import nn, optim
from tqdm import trange
from tensorboardX import SummaryWriter

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import util, logging, time, gc
import numpy as np
from scipy.stats import sem
from numpy import std

from argparse import ArgumentParser

import torchvision
from torch.utils.data import TensorDataset, DataLoader

import os

from gaussian import HyperLayer

logging.basicConfig(filename='run.log',level=logging.INFO)
LOG = logging.getLogger()

"""
Experiment: learn a mapping from a random x, to x sorted.

"""
tbw = SummaryWriter()

util.DEBUG = False
BUCKET_SIGMA = 0.05

def clean(axes=None):

    if axes is None:
        axes = plt.gca()

    [s.set_visible(False) for s in axes.spines.values()]
    axes.tick_params(top=False, bottom=False, left=False, right=False, labelbottom=False, labelleft=False)

def gen(b, data, size):

    t = torch.zeros(b, size, 1, 28, 28)

    # Select random digits
    for i in range(size):
        sample = random.choices(data[i], k=b)
        t[:, i, :, :, :] = torch.cat(sample, dim=0)

    x = t.clone()

    # Shuffle
    for bi in range(b):
        perm = torch.randperm(size)
        x[bi] = x[bi, perm, :, :, :]

    return x, t

def go(arg):
    """

    :param arg:
    :return:
    """

    """
    Load and organize the data
    """
    trans = torchvision.transforms.ToTensor()
    if arg.final:
        train = torchvision.datasets.MNIST(root=arg.data, train=True, download=True, transform=trans)
        trainloader = torch.utils.data.DataLoader(train, batch_size=arg.batch, shuffle=True, num_workers=2)

        test = torchvision.datasets.MNIST(root=arg.data, train=False, download=True, transform=trans)
        testloader = torch.utils.data.DataLoader(test, batch_size=arg.batch, shuffle=False, num_workers=2)

    else:
        NUM_TRAIN = 45000
        NUM_VAL = 5000
        total = NUM_TRAIN + NUM_VAL

        train = torchvision.datasets.MNIST(root=arg.data, train=True, download=True, transform=trans)

        trainloader = DataLoader(train, batch_size=arg.batch, sampler=util.ChunkSampler(0, NUM_TRAIN, total))
        testloader = DataLoader(train, batch_size=arg.batch, sampler=util.ChunkSampler(NUM_TRAIN, NUM_VAL, total))

    shape = (1, 28, 28)
    num_classes = 10

    train = {label: [] for label in range(10)}

    for inps, labels in trainloader:
        b, c, h, w = inps.size()
        for i in range(b):
            image = inps[i:i+1, :, :, :]
            label = labels[i].item()
            train[label].append(image)

    if arg.limit is not None:
        train = {label: imgs[:arg.limit] for label, imgs in train.items()}

    # train = {label: torch.cat(imgs, dim=0) for label, imgs in train}

    test = {label: [] for label in range(10)}
    for inps, labels in trainloader:
        b, c, h, w = inps.size()
        for i in range(b):
            image = inps[i:i+1, :, :, :]
            label = labels[i].item()
            test[label].append(image)

    # train = {label: torch.cat(imgs, dim=0) for label, imgs in train}
    del b, c, h, w

    torch.manual_seed(arg.seed)

    ndots = arg.iterations // arg.dot_every

    results = np.zeros((arg.reps, ndots))

    for r in range(arg.reps):
        print('starting {} out of {} repetitions'.format(r, arg.reps))
        util.makedirs('./mnist-sort/{}'.format( r))

        model = sort.SortLayer(arg.size, additional=arg.additional, sigma_scale=arg.sigma_scale, sigma_floor=arg.min_sigma)

        # bottom = nn.Linear(28*28, 256)
        # tokeys = nn.Sequential(
        #     util.Flatten(),
        #     bottom, nn.ReLU(),
        #     nn.Linear(256, 64), nn.ReLU(),
        #     nn.Linear(64, 1)
        # )

        # - channel sizes
        c1, c2, c3 = 16, 64, 128
        h1, h2 = 256, 128

        tokeys = nn.Sequential(
            nn.Conv2d(1, c1, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c1, c1, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c1, c1, (3, 3), padding=1), nn.ReLU(),
            nn.BatchNorm2d(c1),
            nn.MaxPool2d((2, 2)),
            nn.Conv2d(c1, c2, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c2, c2, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c2, c2, (3, 3), padding=1), nn.ReLU(),
            nn.BatchNorm2d(c2),
            nn.MaxPool2d((2, 2)),
            nn.Conv2d(c2, c3, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c3, c3, (3, 3), padding=1), nn.ReLU(),
            nn.Conv2d(c3, c3, (3, 3), padding=1), nn.ReLU(),
            nn.BatchNorm2d(c3),
            nn.MaxPool2d((2, 2)),
            util.Flatten(),
            nn.Linear(9 * c3, h1), nn.ReLU(),
            nn.Linear(h1, h2), nn.ReLU(),
            nn.Linear(h2, 1), nn.BatchNorm1d(1),
        )

        if arg.cuda:
            model.cuda()
            tokeys.cuda()

        optimizer = optim.Adam(list(model.parameters()) + list(tokeys.parameters()), lr=arg.lr)

        # seen = 0
        # for ep in trange(50):
        #     for i, (batch, labels) in enumerate(trainloader):
        #
        #         optimizer.zero_grad()
        #
        #         labels = labels.float()
        #         keys = tokeys(batch)
        #
        #         loss = F.mse_loss(keys.squeeze(), labels)
        #
        #         loss.backward()
        #
        #         optimizer.step()
        #
        #         seen += batch.size(0)
        #         tbw.add_scalar('mnist-sort/pretrain-loss', loss.data.item(), seen)

        for i in trange(arg.iterations):

            if i > 3000:
                util.DEBUG = True

            x, t = gen(arg.batch, train, arg.size) # torch.randn((arg.batch_size,) + SHAPE)

            t, idxs = x.sort(dim=1)

            if arg.cuda:
                x, t = x.cuda(), t.cuda()
            x, t = Variable(x), Variable(t)

            optimizer.zero_grad()

            keys = tokeys(x.view(arg.batch * arg.size, 1, 28, 28))
            keys = keys.view(arg.batch, arg.size)

            keys.retain_grad()

            x = x.view(arg.batch, arg.size, -1)
            t = t.view(arg.batch, arg.size, -1)

            y, _ = model(x, keys=keys)

            loss = F.mse_loss(y, t) # compute the loss

            loss.backward()
            # perms = util.DEBUG
            #
            # print(perms)
            # print((keys - keys.mean(dim=1, keepdim=True)).sign())
            # print((keys - keys.mean(dim=1, keepdim=True)).sign() - 0.1 *  keys.grad.sign())
            #
            # k =  (keys - keys.mean(dim=1, keepdim=True)).sign()
            # # print( (perms[:, 0] < perms[:, 1]) == (k[:, 0] < k[:, 1]) )

            optimizer.step()

            tbw.add_scalar('mnist-sort/loss/{}/{}'.format(arg.size, r), loss.data.item(), i*arg.batch)

            # Compute test set loss
            if i % arg.plot_every == 0:
                with torch.no_grad():

                    x, t = gen(arg.batch, test, arg.size)
                    if arg.cuda:
                        x, t = x.cuda(), t.cuda()

                    x, t = Variable(x), Variable(t)

                    keys = tokeys(x.view(arg.batch * arg.size, 1, 28, 28))
                    # keys = tokeys(x.view(arg.batch * arg.size, -1))
                    keys = keys.view(arg.batch, arg.size)

                    x = x.view(arg.batch, arg.size, -1)
                    t = t.view(arg.batch, arg.size, -1)

                    yi, _ = model(x, keys=keys, train=False)
                    yt, _ = model(x, keys=keys, train=True)

                    input  = x[0].view(arg.size, 28, 28).data.cpu().numpy()
                    target = t[0].view(arg.size, 28, 28).data.cpu().numpy()
                    output_inf   = yi[0].view(arg.size, 28, 28).data.cpu().numpy()
                    output_train = yt[0].view(arg.size, 28, 28).data.cpu().numpy()

                    plt.figure(figsize=(arg.size, 4))
                    for col in range(arg.size):
                        ax = plt.subplot(4, arg.size, col + 1)
                        ax.imshow(input[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('input')

                        ax = plt.subplot(4, arg.size, col + arg.size + 1)
                        ax.imshow(target[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('target')

                        ax = plt.subplot(4, arg.size, col + arg.size * 2 + 1)
                        ax.imshow(output_inf[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('inference')

                        ax = plt.subplot(4, arg.size, col + arg.size * 3 + 1)
                        ax.imshow(output_train[col], cmap='gray_r')
                        clean(ax)

                        if col == 0:
                            ax.set_ylabel('training')

                    plt.savefig('./mnist-sort/mnist.{:04}.pdf'.format(i))
                    # sys.exit()

            if i % arg.dot_every == 0 and False:
                # print('gradient ', bottom.weight.grad.mean(), bottom.weight.grad.var())

                with torch.no_grad():

                    losses = []
                    for ii in range(10000//arg.batch):
                        x, t = gen(arg.batch, test, arg.size)

                        if arg.cuda:
                            x, t = x.cuda(), t.cuda()

                        x, t = Variable(x), Variable(t)

                        keys = tokeys(x.view(arg.batch * arg.size, 1, 28, 28))
                        #keys = tokeys(x.view(arg.batch * arg.size, -1))
                        keys = keys.view(arg.batch, arg.size)

                        x = x.view(arg.batch, arg.size, -1)
                        t = t.view(arg.batch, arg.size, -1)

                        y, _ = model(x, keys=keys, train=False)

                        loss = F.mse_loss(y, t)  # compute the loss

                        losses.append(loss.item())

                    print('loss', np.mean(losses))

                    results[r, i//arg.dot_every] = np.mean(losses)

                    tbw.add_scalar('mnist-sort/testloss/{}/{}'.format(arg.size, r), np.mean(losses), i * arg.batch)

    np.save('results.{}.np'.format(arg.size), results)
    print('experiments finished')

    plt.figure(figsize=(10, 5))
    ax = plt.gca()

    if results.shape[0] > 1:
        ax.errorbar(x=np.arange(ndots) * arg.dot_every, y=np.mean(results[:, :], axis=0),
                        yerr=np.std(results[:, :], axis=0),
                        label='size {0}x{0}, r={1}'.format(arg.size, arg.reps))
    else:
        ax.plot(np.arange(ndots) * arg.dot_every, np.mean(results[:, :], axis=0),
                        label='size {0}x{0}'.format(arg.size))

    ax.legend()

    util.basic(ax)

    ax.spines['bottom'].set_position('zero')
    ax.set_ylim(0.0, 1.0)
#    ax.set_xlim(0.0, 100.0)

    plt.xlabel('iterations')
    plt.ylabel('error')

    plt.savefig('./quicksort/result.png')
    plt.savefig('./quicksort/result.pdf')

if __name__ == "__main__":

    ## Parse the command line options
    parser = ArgumentParser()

    parser.add_argument("-s", "--size",
                        dest="size",
                        help="Dimensionality of the input.",
                        default=8, type=int)

    parser.add_argument("-b", "--batch-size",
                        dest="batch",
                        help="The batch size.",
                        default=128, type=int)

    parser.add_argument("-i", "--iterations",
                        dest="iterations",
                        help="Number of iterations (in batches).",
                        default=8000, type=int)

    parser.add_argument("-a", "--additional",
                        dest="additional",
                        help="Number of additional points sampled globally",
                        default=2, type=int)

    parser.add_argument("-c", "--cuda", dest="cuda",
                        help="Whether to use cuda.",
                        action="store_true")

    parser.add_argument("-l", "--learn-rate",
                        dest="lr",
                        help="Learning rate",
                        default=0.0001, type=float)

    parser.add_argument("-p", "--plot-every",
                        dest="plot_every",
                        help="Plot every x iterations",
                        default=50, type=int)

    parser.add_argument("-r", "--random-seed",
                        dest="seed",
                        help="Random seed.",
                        default=32, type=int)

    parser.add_argument("-d", "--dot-every",
                        dest="dot_every",
                        help="How many iterations per dot in the loss curves.",
                        default=1000, type=int)

    parser.add_argument("-D", "--data",
                        dest="data",
                        help="Data ditectory.",
                        default='./data', type=str)

    parser.add_argument("-S", "--sigma-scale",
                        dest="sigma_scale",
                        help="Sigma scale.",
                        default=0.1, type=float)

    parser.add_argument("-R", "--repeats",
                        dest="reps",
                        help="Number of repeats.",
                        default=10, type=int)

    parser.add_argument("-M", "--min-sigma",
                        dest="min_sigma",
                        help="Sigma floor (minimum sigma value).",
                        default=0.0, type=float)

    parser.add_argument("-L", "--limit",
                        dest="limit",
                        help="Limit on the nr ofexamples per class (for debugging).",
                        default=None, type=int)

    parser.add_argument("-f", "--final", dest="final",
                        help="Whether to run on the real test set.",
                        action="store_true")

    options = parser.parse_args()

    print('OPTIONS ', options)
    LOG.info('OPTIONS ' + str(options))

    go(options)
<|endoftext|>


<commit_before>import os, glob, re
from colorama import Fore
from colorama import init
import instantmusic

def play(data):
    if len(data[5:]) == 0:
        print(Fore.BLUE + "Song name doesn't exist. (music '"'song name'"') " + Fore.RESET)
    else:
        song = data[6:]
        music = os.system("ls | grep -i " +'"'+ song +'"' +" >> music.txt")
        txt = open("music.txt", "r+")
        isthere = txt.readlines()

        if len(isthere) == 0:
            os.system("instantmusic -s " + song)
            play(data)
        else:
            song = isthere[0]
            song = song.replace("\n", "")
            newname = re.sub(r'\([^)]*\)', '', song).replace("-", "").replace("_", "")
            os.renames(song, newname)
            os.system("xdg-open " + newname.replace(" ", "\ "))
            txt.seek(0)
            txt.truncate()<commit_msg>Update music.py<commit_after>import os, glob, re
from colorama import Fore
from colorama import init
import instantmusic

def play(data):
    if len(data[5:]) == 0:
        print(Fore.BLUE + "Song name doesn't exist. (music '"'song name'"') " + Fore.RESET)
    else:
        song = data[6:]
        music = os.system("ls | grep -i " +'"'+ song +'"' +" >> music.txt")
        txt = open("music.txt", "r+")
        isthere = txt.readlines()

        if len(isthere) == 0:
            os.system("instantmusic -s " + song)
            play(data)
        else:
            song = isthere[0]
            song = song.replace("\n", "")
            newname = re.sub(r'\([^)]*\)', '', song).replace("-", "").replace("_", "").replace("&", "").replace("©", "")
            os.renames(song, newname)
            os.system("xdg-open " + newname.replace(" ", "\ "))
            txt.seek(0)
            txt.truncate()
<|endoftext|>


<commit_before>import os, glob, re
from colorama import Fore
from colorama import init
import instantmusic

def play(data):
    if len(data[5:]) == 0:
        print(Fore.BLUE + "Song name doesn't exist. (music '"'song name'"') " + Fore.RESET)
    else:
        song = data[6:]
        music = os.system("ls | grep -i " +'"'+ song +'"' +" >> music.txt")
        txt = open("music.txt", "r+")
        isthere = txt.readlines()

        if len(isthere) == 0:
            os.system("instantmusic -s " + song)
            play(data)
        else:
            song = isthere[0]
            song = song.replace("\n", "")
            newname = re.sub(r'\([^)]*\)', '', song).replace("-", "").replace("_", "").replace("&", "").replace("©", "")
            os.renames(song, newname)
            os.system("xdg-open " + newname.replace(" ", "\ "))
            txt.seek(0)
            txt.truncate()
<commit_msg>Update music.py<commit_after># -*- coding: utf-8 -*-

import os, glob, re
from colorama import Fore
from colorama import init
import instantmusic

def play(data):
    if len(data[5:]) == 0:
        print(Fore.BLUE + "Song name doesn't exist. (music '"'song name'"') " + Fore.RESET)
    else:
        song = data[6:]
        music = os.system("ls | grep -i " +'"'+ song +'"' +" >> music.txt")
        txt = open("music.txt", "r+")
        isthere = txt.readlines()

        if len(isthere) == 0:
            os.system("instantmusic -s " + song)
            play(data)
        else:
            song = isthere[0]
            song = song.replace("\n", "")
            newname = re.sub(r'\([^)]*\)', '', song).replace("-", "").replace("_", "").replace("&", "").replace("©", "")
            os.renames(song, newname)
            os.system("xdg-open " + newname.replace(" ", "\ "))
            txt.seek(0)
            txt.truncate()
<|endoftext|>


<commit_before>try:
    from src.base_classes import SQLModule
except ImportError:
    from base_classes import SQLModule

class Company(SQLModule):
    table_name = "company"
    columns = ("name", "address", "website", "phone")
    columns_with_uid = ("id", "name", "address", "website", "phone")

    def __init__(self, uid="", name="", address="", website="", phone=""):
        super().__init__()
        self.uid = uid
        self.name = name
        self.address = address
        self.website = website
        self.phone = phone

    @property
    def values(self):
        return self.name, self.address, self.website, self.phone

    @property
    def values_with_id(self):
        return self.name, self.address, self.website, self.phone, self.uid

    def get_all_companies(self):
        if not self.db:
            self.init_db()

        query = "SELECT {} FROM company ORDER BY id;".format(", ".join(Company.columns_with_uid))

        data = self.db.conn.execute(query)

        return [Company(*item) for item in data]

    def get_a_company(self, uid):
        assert isinstance(uid, int)
        if not self.db:
            self.init_db()

        query = "SELECT {} FROM company WHERE (id=?) ORDER BY id;".format(", ".join(Company.columns_with_uid))

        data = self.db.conn.execute(query, (uid,))

        return [Company(*item) for item in data]

    def add_company_to_db(self):
        if not self.db:
            self.init_db()

        # Make sure that the Company does not already exist
        assert self.uid == ""

        self.insert_row_into_db(Company.table_name, Company.columns, self.values)

        # updating the uid for the current object
        self.uid = self.get_all_companies()[-1].uid

    def update_company_in_db(self):
        if not self.db:
            self.init_db()

        # make sure tht the Company does exist in the
        assert not self.uid == ""

        self.update_row_in_db(Company.table_name, Company.columns, self.values_with_id)

    def delete_a_company(self):
        if not self.db:
            self.init_db()

        # Make sure that the Company exists
        assert not self.uid == ""

        self.delete_row_in_db(Company.table_name, self.uid)

    def __str__(self):
        return """
        uid: {}
        name: {}
        address: {}
        website: {}
        phone: {}""".format(self.uid, self.name, self.address, self.website, self.phone)

if __name__ == "__main__":
    test = Company(name="DavidInc")
    test2 = test.get_a_company(7)[0]
    print(test2)
    test2.name = "I just changed you!"
    test2.update_company_in_db()
    for company in test.get_all_companies():
        print(company)
<commit_msg>I can delete companies! Next I have to this for the other classes<commit_after>try:
    from src.base_classes import SQLModule
except ImportError:
    from base_classes import SQLModule

class Company(SQLModule):
    table_name = "company"
    columns = ("name", "address", "website", "phone")
    columns_with_uid = ("id", "name", "address", "website", "phone")

    def __init__(self, uid="", name="", address="", website="", phone=""):
        super().__init__()
        self.uid = uid
        self.name = name
        self.address = address
        self.website = website
        self.phone = phone

    @property
    def values(self):
        return self.name, self.address, self.website, self.phone

    @property
    def values_with_id(self):
        return self.name, self.address, self.website, self.phone, self.uid

    def get_all_companies(self):
        if not self.db:
            self.init_db()

        query = "SELECT {} FROM company ORDER BY id;".format(", ".join(Company.columns_with_uid))

        data = self.db.conn.execute(query)

        return [Company(*item) for item in data]

    def get_a_company(self, uid):
        assert isinstance(uid, int)
        if not self.db:
            self.init_db()

        query = "SELECT {} FROM company WHERE (id=?) ORDER BY id;".format(", ".join(Company.columns_with_uid))

        data = self.db.conn.execute(query, (uid,))

        return [Company(*item) for item in data]

    def add_company_to_db(self):
        if not self.db:
            self.init_db()

        # Make sure that the Company does not already exist
        assert self.uid == ""

        self.insert_row_into_db(Company.table_name, Company.columns, self.values)

        # updating the uid for the current object
        self.uid = self.get_all_companies()[-1].uid

    def update_company_in_db(self):
        if not self.db:
            self.init_db()

        # make sure tht the Company does exist in the
        assert not self.uid == ""

        self.update_row_in_db(Company.table_name, Company.columns, self.values_with_id)

    def delete_company_in_db(self):
        if not self.db:
            self.init_db()

        # Make sure that the Company exists
        assert not self.uid == ""

        self.delete_row_in_db(Company.table_name, (self.uid,))

    def __str__(self):
        return """
        uid: {}
        name: {}
        address: {}
        website: {}
        phone: {}""".format(self.uid, self.name, self.address, self.website, self.phone)

if __name__ == "__main__":
    test = Company(name="DavidInc")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2014      Oleg Plakhotniuk
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.tools.capabilities.bank.transactions import \
    AmericanTransaction as AmTr
from weboob.capabilities.shop import Order, Payment, Item
from weboob.browser.pages import HTMLPage, pagination, NextPage

from datetime import datetime
from decimal import Decimal
import re

class AmazonPage(HTMLPage):
    @property
    def logged(self):
        return bool(self.doc.xpath(u'//*[contains(text(),"Sign Out")]'))

class HomePage(AmazonPage):
    def to_login(self):
        url1 = self.doc.xpath('//a[@id="nav-link-yourAccount"]/@href')
        url2 = self.doc.xpath('//a[@id="nav-your-account"]/@href')
        self.browser.location((url1 or url2)[0])
        return self.browser.page

class LoginPage(AmazonPage):
    def login(self, email, password):
        form = self.get_form(name='signIn')
        form['email'] = email
        form['password'] = password
        form.submit()

class HistoryPage(AmazonPage):
    def iter_years(self):
        for year in self.opt_years():
            yield self.to_year(year)

    @pagination
    def iter_orders(self):
        for id_ in self.doc.xpath(
                     u'//span[contains(text(),"Order #")]/../span[2]/text()'):
            yield self.browser.to_order(id_.strip())
        for next_ in self.doc.xpath(u'//ul[@class="a-pagination"]'
                                    u'//a[contains(text(),"Next")]/@href'):
            raise NextPage(next_)

    def to_year(self, year):
        form = self.get_form('//form[contains(@class,"time-period-chooser")]')
        form['orderFilter'] = [year]
        form.submit()
        return self.browser.page

    def opt_years(self):
        return [x for x in self.doc.xpath(
                        '//select[@name="orderFilter"]/option/@value'
                ) if x.startswith('year-')]

class OrderPage(AmazonPage):
    def shouldSkip(self):
        # Reports only fully shipped and delivered orders, because they have
        # finalized payment amounts.
        # Payment for not yet shipped orders may change, and is not always
        # available.
        return bool([x for s in [u'Not Yet Shipped', u'Not yet shipped',
            u'Preparing for Shipment', u'Shipping now', u'In transit',
            u'On the way']
            for x in self.doc.xpath(u'//*[contains(text(),"%s")]' % s)])

class OrderNewPage(OrderPage):
    is_here = u'//*[contains(text(),"Ordered on")]'

    def order(self):
        if not self.shouldSkip():
            order = Order(id=self.order_number())
            order.date = self.order_date()
            order.tax = self.tax()
            order.discount = self.discount()
            order.shipping = self.shipping()
            return order

    def order_date(self):
        return datetime.strptime(
            re.match('.*Ordered on ([^ ]+ [0-9]+, [0-9]+) .*',
                     self.date_num()).group(1),
            '%B %d, %Y')

    def order_number(self):
        m = re.match('.*Order# +([^ ]+) .*', self.date_num())
        if m:
            return m.group(1)

    def payments(self):
        if self.gift():
            pmt = Payment()
            pmt.date = self.order_date()
            pmt.method = u'GIFT CARD'
            pmt.amount = -self.gift()
            yield pmt
        transactions = list(self.transactions())
        if transactions:
            for t in transactions:
                yield t
        else:
            for method in self.paymethods():
                pmt = Payment()
                pmt.date = self.order_date()
                pmt.method = method
                pmt.amount = self.grand_total()
                yield pmt
                break

    def paymethods(self):
        for root in self.doc.xpath('//h5[contains(text(),"Payment Method")]'):
            alt = root.xpath('../div/img/@alt')[0]
            span = root.xpath('../div/span/text()')[0]
            digits = re.match(r'[^0-9]*([0-9]+)[^0-9]*', span).group(1)
            yield u'%s %s' % (alt, digits)

    def grand_total(self):
        return AmTr.decimal_amount(self.doc.xpath(
            u'//span[contains(text(),"Grand Total:")]/..'
            u'/following-sibling::div[1]/span/text()')[0].strip())

    def date_num(self):
        return u' '.join(self.doc.xpath(
            '//span[@class="order-date-invoice-item"]/text()'
            )).replace('\n', '')

    def tax(self):
        return self.amount(u'Estimated tax to be collected')

    def shipping(self):
        return self.amount(u'Free shipping', u'Free Shipping',
                           u'Shipping & Handling')

    def discount(self):
        return self.amount(u'Promotion applied', u'Promotion Applied',
                           u'Subscribe & Save', u'Your Coupon Savings',
                           u'Lightning Deal')

    def gift(self):
        return self.amount(u'Gift Card Amount')

    def amount(self, *names):
        return Decimal(sum(AmTr.decimal_amount(amount.strip())
           for n in names for amount in self.doc.xpath(
           '(//span[contains(text(),"%s:")]/../..//span)[2]/text()' % n)))

    def transactions(self):
        for row in self.doc.xpath('//span[contains(text(),"Transactions")]'
                                  '/../../div/div'):
            text = row.text_content().strip().replace('\n', ' ')
            if u'Items shipped:' not in text:
                continue
            date, method, amount = re.match(
                '.* '     '([A-z]+ [0-9]+, [0-9]+)'
                '[ -]+'   '([A-z][^:]+)'
                ': +'     '([^ ]+)', text).groups()
            date = datetime.strptime(date, '%B %d, %Y')
            method = method.replace(u'ending in ', u'').upper()
            amount = AmTr.decimal_amount(amount)
            pmt = Payment()
            pmt.date = date
            pmt.method = method
            pmt.amount = amount
            yield pmt

    def items(self):
        for item in self.doc.xpath('//div[contains(@class,"a-box shipment")]'
                                   '/div/div/div/div/div/div'):
            url = (item.xpath(u'*//a[contains(@href,"/gp/product")]/@href') +
                   [u''])[0]
            label = u''.join(item.xpath(
                '*//a[contains(@href,"/gp/product")]/text()')).strip()
            price = u''.join(x.strip() for x in item.xpath(
                '*//span[contains(text(),"$")]/text()')
                if x.strip().startswith('$'))
            price = AmTr.decimal_amount(price)
            multi = re.match(u'([0-9]+) of (.*)', label)
            if multi:
                amount, label = multi.groups()
                price *= Decimal(amount)
            if url:
                url = unicode(self.browser.BASEURL) + \
                      re.match(u'(/gp/product/.*)/ref=.*', url).group(1)
            if label and price:
                itm = Item()
                itm.label = label
                itm.url = url
                itm.price = price
                yield itm

class OrderOldPage(OrderPage):
    is_here = u'//*[contains(text(),"Amazon.com order number")]'

    def order(self):
        if not self.shouldSkip():
            order = Order(id=self.order_number())
            order.date = self.order_date()
            order.tax = self.tax()
            order.discount = self.discount()
            order.shipping = self.shipping()
            return order

    def order_date(self):
        return datetime.strptime(u' '.join(self.doc.xpath(
            u'//b[contains(text(),"Order Placed")]/../text()')).strip(),
            '%B %d, %Y')

    def order_number(self):
        return u' '.join(self.doc.xpath(
            u'//td/b[contains(text(),"Amazon.com order number")]/../text()')
        ).strip()

    def tax(self):
        return self.sum_amounts(u'Sales Tax:')

    def discount(self):
        return self.sum_amounts(u'Subscribe & Save:', u'Promotion applied:',
                            u'Promotion Applied:', u'Your Coupon Savings:')

    def shipping(self):
        return self.sum_amounts(u'Shipping & Handling:', u'Free shipping:',
                                u'Free Shipping:')

    def payments(self):
        for shmt in self.shipments():
            gift = self.gift(shmt)
            if gift:
                pmt = Payment()
                pmt.date = self.order_date()
                pmt.method = u'GIFT CARD'
                pmt.amount = -gift
                yield pmt
        transactions = list(self.transactions())
        if transactions:
            for t in transactions:
                yield t
        else:
            for method in self.paymethods():
                pmt = Payment()
                pmt.date = self.order_date()
                pmt.method = method
                pmt.amount = self.grand_total()
                yield pmt
                break

    def shipments(self):
        for cue in (u'Shipment #', u'Subscribe and Save Shipment'):
            for shmt in self.doc.xpath('//b[contains(text(),"%s")]' % cue):
                yield shmt

    def items(self):
        for shmt in self.shipments():
            root = shmt.xpath(u'../../../../../../../..'
                              u'//b[text()="Items Ordered"]')[0]
            for item in root.xpath('../../../tr')[1:]:
                count = url = label = None
                for div in item.xpath('*//div'):
                    m = re.match(u'^\s*(\d+)\s*of:(.*)$', div.text,
                                 re.MULTILINE + re.DOTALL)
                    if not m:
                        continue
                    count = Decimal(m.group(1).strip())
                    label = unicode(m.group(2).strip())
                    if label:
                        url = u''
                    else:
                        a = div.xpath('*//a[contains(@href,"/gp/product")]')[0]
                        url = unicode(a.attrib['href'])
                        label = unicode(a.text.strip())
                price1 = item.xpath('*//div')[-1].text.strip()
                price = count * AmTr.decimal_amount(price1)

                itm = Item()
                itm.label = label
                itm.url = url
                itm.price = price
                yield itm

    def sum_amounts(self, *names):
        return sum(self.amount(shmt,x) for shmt in self.shipments()
                                       for x in names)

    def amount(self, shmt, name):
        for root in shmt.xpath(u'../../../../../../../..'
                               u'//td[text()="Item(s) Subtotal: "]/../..'):
            for node in root.xpath(u'tr/td[text()="%s"]' % name):
                return AmTr.decimal_amount(
                    node.xpath('../td')[-1].text.strip())
            for node in root.xpath(u'tr/td/b[text()="%s"]' % name):
                return AmTr.decimal_amount(
                    node.xpath('../../td/b')[-1].text.strip())
        return Decimal(0)

    def gift(self, shmt):
        return self.amount(shmt, u'Gift Card Amount:')

    def paymethods(self):
        root = self.doc.xpath('//b[text()="Payment Method: "]/..')[0]
        text = root.text_content().strip()
        while text:
            for pattern in [
                    u'^.*Payment Method:',
                    u'^([^\n]+)\n +\| Last digits: +([0-9]+)\n',
                    u'^Gift Card\n', # Skip gift card.
                    u'^Billing address.*$']:
                match = re.match(pattern, text, re.DOTALL+re.MULTILINE)
                if match:
                    text = text[match.end():].strip()
                    if match.groups():
                        yield u' '.join(match.groups()).upper()
                    break
            else:
                break

    def transactions(self):
        for tr in self.doc.xpath(
                u'//div[contains(b,"Credit Card transactions")]'
                u'/following-sibling::table[1]/tr'):
            label, date = tr.xpath('td[1]/text()')[0].strip().split(u'\xa0')
            amount = tr.xpath('td[2]/text()')[0].strip()
            date = datetime.strptime(date, '%B %d, %Y:')
            method = label.replace(u'ending in ', u'')[:-1].upper()
            amount = AmTr.decimal_amount(amount)
            pmt = Payment()
            pmt.date = date
            pmt.method = method
            pmt.amount = amount
            yield pmt

    def grand_total(self):
        return AmTr.decimal_amount(self.doc.xpath(
            '//td[b="Grand Total:"]/following-sibling::td[1]/b')[0].text)
<commit_msg>Support new version of Amazon order details header. Fixes #1721<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2014      Oleg Plakhotniuk
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.tools.capabilities.bank.transactions import \
    AmericanTransaction as AmTr
from weboob.capabilities.shop import Order, Payment, Item
from weboob.browser.pages import HTMLPage, pagination, NextPage

from datetime import datetime
from decimal import Decimal
import re

class AmazonPage(HTMLPage):
    @property
    def logged(self):
        return bool(self.doc.xpath(u'//*[contains(text(),"Sign Out")]'))

class HomePage(AmazonPage):
    def to_login(self):
        url1 = self.doc.xpath('//a[@id="nav-link-yourAccount"]/@href')
        url2 = self.doc.xpath('//a[@id="nav-your-account"]/@href')
        self.browser.location((url1 or url2)[0])
        return self.browser.page

class LoginPage(AmazonPage):
    def login(self, email, password):
        form = self.get_form(name='signIn')
        form['email'] = email
        form['password'] = password
        form.submit()

class HistoryPage(AmazonPage):
    def iter_years(self):
        for year in self.opt_years():
            yield self.to_year(year)

    @pagination
    def iter_orders(self):
        for id_ in self.doc.xpath(
                     u'//span[contains(text(),"Order #")]/../span[2]/text()'):
            yield self.browser.to_order(id_.strip())
        for next_ in self.doc.xpath(u'//ul[@class="a-pagination"]'
                                    u'//a[contains(text(),"Next")]/@href'):
            raise NextPage(next_)

    def to_year(self, year):
        form = self.get_form('//form[contains(@class,"time-period-chooser")]')
        form['orderFilter'] = [year]
        form.submit()
        return self.browser.page

    def opt_years(self):
        return [x for x in self.doc.xpath(
                        '//select[@name="orderFilter"]/option/@value'
                ) if x.startswith('year-')]

class OrderPage(AmazonPage):
    def shouldSkip(self):
        # Reports only fully shipped and delivered orders, because they have
        # finalized payment amounts.
        # Payment for not yet shipped orders may change, and is not always
        # available.
        return bool([x for s in [u'Not Yet Shipped', u'Not yet shipped',
            u'Preparing for Shipment', u'Shipping now', u'In transit',
            u'On the way']
            for x in self.doc.xpath(u'//*[contains(text(),"%s")]' % s)])

class OrderNewPage(OrderPage):
    is_here = u'//*[contains(text(),"Ordered on")]'

    def order(self):
        if not self.shouldSkip():
            order = Order(id=self.order_number())
            order.date = self.order_date()
            order.tax = self.tax()
            order.discount = self.discount()
            order.shipping = self.shipping()
            return order

    def order_date(self):
        return datetime.strptime(
            re.match('.*Ordered on ([^ ]+ [0-9]+, [0-9]+) .*',
                     self.date_num()).group(1),
            '%B %d, %Y')

    def order_number(self):
        m = re.match('.*Order# +([^ ]+) .*', self.date_num())
        if m:
            return m.group(1)

    def payments(self):
        if self.gift():
            pmt = Payment()
            pmt.date = self.order_date()
            pmt.method = u'GIFT CARD'
            pmt.amount = -self.gift()
            yield pmt
        transactions = list(self.transactions())
        if transactions:
            for t in transactions:
                yield t
        else:
            for method in self.paymethods():
                pmt = Payment()
                pmt.date = self.order_date()
                pmt.method = method
                pmt.amount = self.grand_total()
                yield pmt
                break

    def paymethods(self):
        for root in self.doc.xpath('//h5[contains(text(),"Payment Method")]'):
            alt = root.xpath('../div/img/@alt')[0]
            span = root.xpath('../div/span/text()')[0]
            digits = re.match(r'[^0-9]*([0-9]+)[^0-9]*', span).group(1)
            yield u'%s %s' % (alt, digits)

    def grand_total(self):
        return AmTr.decimal_amount(self.doc.xpath(
            u'//span[contains(text(),"Grand Total:")]/..'
            u'/following-sibling::div[1]/span/text()')[0].strip())

    def date_num(self):
        return u' '.join(self.doc.xpath(
            '//span[@class="order-date-invoice-item"]/text()'
            ) or self.doc.xpath(
            '//*[contains(text(),"Ordered on")]/text()')).replace('\n', '')

    def tax(self):
        return self.amount(u'Estimated tax to be collected')

    def shipping(self):
        return self.amount(u'Free shipping', u'Free Shipping',
                           u'Shipping & Handling')

    def discount(self):
        return self.amount(u'Promotion applied', u'Promotion Applied',
                           u'Subscribe & Save', u'Your Coupon Savings',
                           u'Lightning Deal')

    def gift(self):
        return self.amount(u'Gift Card Amount')

    def amount(self, *names):
        return Decimal(sum(AmTr.decimal_amount(amount.strip())
           for n in names for amount in self.doc.xpath(
           '(//span[contains(text(),"%s:")]/../..//span)[2]/text()' % n)))

    def transactions(self):
        for row in self.doc.xpath('//span[contains(text(),"Transactions")]'
                                  '/../../div/div'):
            text = row.text_content().strip().replace('\n', ' ')
            if u'Items shipped:' not in text:
                continue
            date, method, amount = re.match(
                '.* '     '([A-z]+ [0-9]+, [0-9]+)'
                '[ -]+'   '([A-z][^:]+)'
                ': +'     '([^ ]+)', text).groups()
            date = datetime.strptime(date, '%B %d, %Y')
            method = method.replace(u'ending in ', u'').upper()
            amount = AmTr.decimal_amount(amount)
            pmt = Payment()
            pmt.date = date
            pmt.method = method
            pmt.amount = amount
            yield pmt

    def items(self):
        for item in self.doc.xpath('//div[contains(@class,"a-box shipment")]'
                                   '/div/div/div/div/div/div'):
            url = (item.xpath(u'*//a[contains(@href,"/gp/product")]/@href') +
                   [u''])[0]
            label = u''.join(item.xpath(
                '*//a[contains(@href,"/gp/product")]/text()')).strip()
            price = u''.join(x.strip() for x in item.xpath(
                '*//span[contains(text(),"$")]/text()')
                if x.strip().startswith('$'))
            price = AmTr.decimal_amount(price)
            multi = re.match(u'([0-9]+) of (.*)', label)
            if multi:
                amount, label = multi.groups()
                price *= Decimal(amount)
            if url:
                url = unicode(self.browser.BASEURL) + \
                      re.match(u'(/gp/product/.*)/ref=.*', url).group(1)
            if label and price:
                itm = Item()
                itm.label = label
                itm.url = url
                itm.price = price
                yield itm

class OrderOldPage(OrderPage):
    is_here = u'//*[contains(text(),"Amazon.com order number")]'

    def order(self):
        if not self.shouldSkip():
            order = Order(id=self.order_number())
            order.date = self.order_date()
            order.tax = self.tax()
            order.discount = self.discount()
            order.shipping = self.shipping()
            return order

    def order_date(self):
        return datetime.strptime(u' '.join(self.doc.xpath(
            u'//b[contains(text(),"Order Placed")]/../text()')).strip(),
            '%B %d, %Y')

    def order_number(self):
        return u' '.join(self.doc.xpath(
            u'//td/b[contains(text(),"Amazon.com order number")]/../text()')
        ).strip()

    def tax(self):
        return self.sum_amounts(u'Sales Tax:')

    def discount(self):
        return self.sum_amounts(u'Subscribe & Save:', u'Promotion applied:',
                            u'Promotion Applied:', u'Your Coupon Savings:')

    def shipping(self):
        return self.sum_amounts(u'Shipping & Handling:', u'Free shipping:',
                                u'Free Shipping:')

    def payments(self):
        for shmt in self.shipments():
            gift = self.gift(shmt)
            if gift:
                pmt = Payment()
                pmt.date = self.order_date()
                pmt.method = u'GIFT CARD'
                pmt.amount = -gift
                yield pmt
        transactions = list(self.transactions())
        if transactions:
            for t in transactions:
                yield t
        else:
            for method in self.paymethods():
                pmt = Payment()
                pmt.date = self.order_date()
                pmt.method = method
                pmt.amount = self.grand_total()
                yield pmt
                break

    def shipments(self):
        for cue in (u'Shipment #', u'Subscribe and Save Shipment'):
            for shmt in self.doc.xpath('//b[contains(text(),"%s")]' % cue):
                yield shmt

    def items(self):
        for shmt in self.shipments():
            root = shmt.xpath(u'../../../../../../../..'
                              u'//b[text()="Items Ordered"]')[0]
            for item in root.xpath('../../../tr')[1:]:
                count = url = label = None
                for div in item.xpath('*//div'):
                    m = re.match(u'^\s*(\d+)\s*of:(.*)$', div.text,
                                 re.MULTILINE + re.DOTALL)
                    if not m:
                        continue
                    count = Decimal(m.group(1).strip())
                    label = unicode(m.group(2).strip())
                    if label:
                        url = u''
                    else:
                        a = div.xpath('*//a[contains(@href,"/gp/product")]')[0]
                        url = unicode(a.attrib['href'])
                        label = unicode(a.text.strip())
                price1 = item.xpath('*//div')[-1].text.strip()
                price = count * AmTr.decimal_amount(price1)

                itm = Item()
                itm.label = label
                itm.url = url
                itm.price = price
                yield itm

    def sum_amounts(self, *names):
        return sum(self.amount(shmt,x) for shmt in self.shipments()
                                       for x in names)

    def amount(self, shmt, name):
        for root in shmt.xpath(u'../../../../../../../..'
                               u'//td[text()="Item(s) Subtotal: "]/../..'):
            for node in root.xpath(u'tr/td[text()="%s"]' % name):
                return AmTr.decimal_amount(
                    node.xpath('../td')[-1].text.strip())
            for node in root.xpath(u'tr/td/b[text()="%s"]' % name):
                return AmTr.decimal_amount(
                    node.xpath('../../td/b')[-1].text.strip())
        return Decimal(0)

    def gift(self, shmt):
        return self.amount(shmt, u'Gift Card Amount:')

    def paymethods(self):
        root = self.doc.xpath('//b[text()="Payment Method: "]/..')[0]
        text = root.text_content().strip()
        while text:
            for pattern in [
                    u'^.*Payment Method:',
                    u'^([^\n]+)\n +\| Last digits: +([0-9]+)\n',
                    u'^Gift Card\n', # Skip gift card.
                    u'^Billing address.*$']:
                match = re.match(pattern, text, re.DOTALL+re.MULTILINE)
                if match:
                    text = text[match.end():].strip()
                    if match.groups():
                        yield u' '.join(match.groups()).upper()
                    break
            else:
                break

    def transactions(self):
        for tr in self.doc.xpath(
                u'//div[contains(b,"Credit Card transactions")]'
                u'/following-sibling::table[1]/tr'):
            label, date = tr.xpath('td[1]/text()')[0].strip().split(u'\xa0')
            amount = tr.xpath('td[2]/text()')[0].strip()
            date = datetime.strptime(date, '%B %d, %Y:')
            method = label.replace(u'ending in ', u'')[:-1].upper()
            amount = AmTr.decimal_amount(amount)
            pmt = Payment()
            pmt.date = date
            pmt.method = method
            pmt.amount = amount
            yield pmt

    def grand_total(self):
        return AmTr.decimal_amount(self.doc.xpath(
            '//td[b="Grand Total:"]/following-sibling::td[1]/b')[0].text)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2013      Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.job import ICapJob
from weboob.tools.ordereddict import OrderedDict
from weboob.tools.value import Value
from .browser import ApecBrowser
from .job import ApecJobAdvert

__all__ = ['ApecBackend']

class ApecBackend(BaseBackend, ICapJob):
    NAME = 'apec'
    DESCRIPTION = u'apec website'
    MAINTAINER = u'Bezleputh'
    EMAIL = 'carton_ben@yahoo.fr'
    VERSION = '0.h'

    BROWSER = ApecBrowser

    places_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '700': u'Alsace',
        '701': u'Aquitaine',
        '702': u'Auvergne',
        '703': u'Basse-Normandie',
        '704': u'Bourgogne',
        '705': u'Bretagne',
        '706': u'Centre',
        '707': u'Champagne',
        '20': u'Corse',
        '99712': u'France Outre-Mer',
        '709': u'Franche-Comté',
        '710': u'Haute-Normandie',
        '711': u'Ile-de-France',
        '712': u'Languedoc-Roussillon',
        '713': u'Limousin',
        '714': u'Lorraine',
        '715': u'Midi-Pyrénées',
        '716': u'Nord-Pas-de-Calais',
        '720': u'PACA',
        '717': u'Pays de La Loire',
        '718': u'Picardie',
        '719': u'Poitou-Charentes',
        '721': u'Rhône-Alpes',
        '99109': u'Allemagne',
        '99106': u'Estonie',
        '99108': u'Lituanie',
        '99116': u'République Tchèque',
        '99110': u'Autriche',
        '99105': u'Finlande',
        '99137': u'Luxembourg',
        '99114': u'Roumanie',
        '99131': u'Belgique',
        '99126': u'Grèce',
        '99144': u'Malte',
        '99132': u'Royaume Uni',
        '99111': u'Bulgarie',
        '99112': u'Hongrie',
        '99135': u'Pays Bas',
        '99117': u'Slovaquie',
        '99254': u'Chypre',
        '99136': u'Irlande',
        '99122': u'Pologne',
        '99145': u'Slovénie',
        '99101': u'Danemark',
        '99127': u'Italie',
        '99139': u'Portugal',
        '99104': u'Suède',
        '99134': u'Espagne',
        '99107': u'Lettonie',
        '99700': u'UE Hors France',
        '99702': u'Amérique du Nord',
        '99715': u'Afrique',
        '99711': u'Océanie',
        '99701': u'Europe Hors UE',
        '99714': u'Amérique Latine',
        '99716': u'Asie',
    }.iteritems())])

    fonction_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101828': u'Commercial, Marketing',
        '101782': u'.....Administration des ventes et SAV',
        '101783': u'.....Chargé d\'affaires, technico-commercial',
        '101784': u'.....Commercial',
        '101785': u'.....Commerce international',
        '101786': u'.....Direction commerciale et marketing',
        '101787': u'.....Direction régionale et d\'agence',
        '101788': u'.....Marketing',
        '101789': u'.....Ventes en magasin',
        '101829': u'Communication, Création',
        '101790': u'.....Communication',
        '101791': u'.....Création',
        '101792': u'.....Documentation, rédaction technique',
        '101793': u'.....Journalisme, édition',
        '101830': u'Direction d\'entreprise',
        '101794': u'.....Adjoint, conseil de direction',
        '101795': u'.....Direction générale',
        '101831': u'Etudes, Recherche et Développement',
        '101796': u'.....Conception, recherche',
        '101797': u'.....Direction recherche et développement',
        '101798': u'.....Etudes socio-économiques',
        '101799': u'.....Projets scientifiques et techniques',
        '101800': u'.....Test, essai, validation, expertise',
        '101832': u'Gestion, Finance, Administration',
        '101801': u'.....Administration, gestion, organisation',
        '101802': u'.....Comptabilité',
        '101803': u'.....Contrôle de gestion, audit',
        '101804': u'.....Direction gestion, finance',
        '101805': u'.....Droit, fiscalité',
        '101806': u'.....Finance, trésorerie',
        '101833': u'Informatique',
        '101807': u'.....Direction informatique',
        '101808': u'.....Exploitation, maintenance informatique',
        '101809': u'.....Informatique de gestion',
        '101810': u'.....Informatique industrielle',
        '101811': u'.....Informatique web, sites et portails Internet',
        '101812': u'.....Maîtrise d\'ouvrage et fonctionnel',
        '101813': u'.....Système, réseaux, données',
        '101834': u'Production Industrielle, Travaux, Chantiers',
        '101814': u'.....Cadres de chantier',
        '101815': u'.....Cadres de production industrielle',
        '101816': u'.....Direction d\'unité industrielle',
        '101835': u'Ressources Humaines',
        '101817': u'.....Administration des RH',
        '101818': u'.....Développement des RH',
        '101819': u'.....Direction des ressources humaines',
        '101820': u'.....Formation initiale et continue',
        '101836': u'Sanitaire, Social, Culture',
        '101821': u'.....Activités sanitaires, sociales et culturelles',
        '101837': u'Services Techniques',
        '101822': u'.....Achats',
        '101823': u'.....Direction des services techniques',
        '101824': u'.....Logistique',
        '101825': u'.....Maintenance, sécurité',
        '101826': u'.....Process, méthodes',
        '101827': u'.....Qualité',
    }.iteritems())])

    secteur_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101752': u'Activités des organisations associatives et administration publique',
        '101753': u'Activités informatiques',
        '101754': u'Activités juridiques et comptables',
        '101755': u'Agroalimentaire',
        '101756': u'Automobile, aéronautique et autres matériels de transport',
        '101757': u'Banque et Assurances',
        '101758': u'Bois - Papier - Imprimerie',
        '101759': u'Chimie - Caoutchouc - Plastique',
        '101760': u'Commerce interentreprises',
        '101761': u'Communication et médias',
        '101762': u'Conseil et gestion des entreprises',
        '101763': u'Construction',
        '101764': u'Distribution généraliste et spécialisée',
        '101765': u'Energies - Eau',
        '101766': u'Equipements électriques et électroniques',
        '101767': u'Formation initiale et continue',
        '101768': u'Gestion des déchets',
        '101769': u'Hôtellerie - Restauration - Loisirs',
        '101770': u'Immobilier',
        '101771': u'Industrie pharmaceutique',
        '101772': u'Ingénierie - R et D',
        '101773': u'Intermédiaires du recrutement',
        '101774': u'Mécanique - Métallurgie',
        '101775': u'Meuble, Textile et autres industries manufacturières',
        '101776': u'Santé - action sociale',
        '101777': u'Services divers aux entreprises',
        '101778': u'Télécommunications',
        '101779': u'Transports et logistique',
    }.iteritems())])

    type_contrat_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101888': u'CDI',
        '101887': u'CDD',
        '101889': u'Travail Temporaire',
    }.iteritems())])

    salary_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101839': u'Moins de 35 K€',
        '101840': u'Entre 35 et 49 K€',
        '101841': u'Entre 50 et 69 K€',
        '101842': u'Entre 70 et 90 K€',
        '101843': u'Plus de 90 K€',
    }.iteritems())])

    date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101850': u'Aujourd\'hui',
        '101851': u'Les 7 derniers jours',
        '101852': u'Les 30 derniers jours',
        '101853': u'Toutes les offres',
    }.iteritems())])

    level_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101846': u'Débutant',
        '101848': u'Expérimenté',
    }.iteritems())])

    CONFIG = BackendConfig(Value('place', label=u'Lieu', choices=places_choices, default=''),
                           Value('fonction', label=u'Fonction', choices=fonction_choices, default=''),
                           Value('secteur', label=u'Secteur', choices=secteur_choices, default=''),
                           Value('contrat', label=u'Contrat', choices=type_contrat_choices, default=''),
                           Value('salaire', label=u'Salaire', choices=salary_choices, default=''),
                           Value('limit_date', label=u'Date', choices=date_choices, default=''),
                           Value('level', label=u'Expérience', choices=level_choices, default=''))

    def search_job(self, pattern=None):
        with self.browser:
            for job_advert in self.browser.search_job(pattern=pattern):
                yield job_advert

    def advanced_search_job(self):
        for job_advert in self.browser.advanced_search_job(region=self.config['place'].get(),
                                                           fonction=self.config['fonction'].get(),
                                                           secteur=self.config['secteur'].get(),
                                                           salaire=self.config['salaire'].get(),
                                                           contrat=self.config['contrat'].get(),
                                                           limit_date=self.config['limit_date'].get(),
                                                           level=self.config['level'].get()):
            yield job_advert

    def get_job_advert(self, _id, advert=None):
        with self.browser:
            return self.browser.get_job_advert(_id, advert)

    def fill_obj(self, advert, fields):
        self.get_job_advert(advert.id, advert)

    OBJECTS = {ApecJobAdvert: fill_obj}
<commit_msg>[Apec] fix choices order display<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2013      Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.job import ICapJob
from weboob.tools.ordereddict import OrderedDict
from weboob.tools.value import Value
from .browser import ApecBrowser
from .job import ApecJobAdvert

__all__ = ['ApecBackend']

class ApecBackend(BaseBackend, ICapJob):
    NAME = 'apec'
    DESCRIPTION = u'apec website'
    MAINTAINER = u'Bezleputh'
    EMAIL = 'carton_ben@yahoo.fr'
    VERSION = '0.h'

    BROWSER = ApecBrowser

    places_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        '00| ': u'-- Indifférent --',
        '01|700': u'Alsace',
        '02|701': u'Aquitaine',
        '03|702': u'Auvergne',
        '04|703': u'Basse-Normandie',
        '05|704': u'Bourgogne',
        '06|705': u'Bretagne',
        '07|706': u'Centre',
        '08|707': u'Champagne',
        '09|20': u'Corse',
        '10|99712': u'France Outre-Mer',
        '11|709': u'Franche-Comté',
        '12|710': u'Haute-Normandie',
        '13|711': u'Ile-de-France',
        '14|712': u'Languedoc-Roussillon',
        '15|713': u'Limousin',
        '16|714': u'Lorraine',
        '17|715': u'Midi-Pyrénées',
        '18|716': u'Nord-Pas-de-Calais',
        '19|720': u'PACA',
        '20|717': u'Pays de La Loire',
        '21|718': u'Picardie',
        '22|719': u'Poitou-Charentes',
        '23|721': u'Rhône-Alpes',
        '24|99109': u'Allemagne',
        '25|99106': u'Estonie',
        '26|99108': u'Lituanie',
        '27|99116': u'République Tchèque',
        '28|99110': u'Autriche',
        '29|99105': u'Finlande',
        '30|99137': u'Luxembourg',
        '31|99114': u'Roumanie',
        '32|99131': u'Belgique',
        '33|99126': u'Grèce',
        '34|99144': u'Malte',
        '35|99132': u'Royaume Uni',
        '36|99111': u'Bulgarie',
        '37|99112': u'Hongrie',
        '38|99135': u'Pays Bas',
        '39|99117': u'Slovaquie',
        '40|99254': u'Chypre',
        '41|99136': u'Irlande',
        '42|99122': u'Pologne',
        '43|99145': u'Slovénie',
        '44|99101': u'Danemark',
        '45|99127': u'Italie',
        '46|99139': u'Portugal',
        '47|99104': u'Suède',
        '48|99134': u'Espagne',
        '49|99107': u'Lettonie',
        '50|99700': u'UE Hors France',
        '51|99702': u'Amérique du Nord',
        '52|99715': u'Afrique',
        '53|99711': u'Océanie',
        '54|99701': u'Europe Hors UE',
        '55|99714': u'Amérique Latine',
        '56|99716': u'Asie',
    }.iteritems())])

    fonction_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        '00| ': u'-- Indifférent --',
        '01|101828': u'Commercial, Marketing',
        '02|101782': u'.....Administration des ventes et SAV',
        '03|101783': u'.....Chargé d\'affaires, technico-commercial',
        '04|101784': u'.....Commercial',
        '05|101785': u'.....Commerce international',
        '06|101786': u'.....Direction commerciale et marketing',
        '07|101787': u'.....Direction régionale et d\'agence',
        '08|101788': u'.....Marketing',
        '09|101789': u'.....Ventes en magasin',
        '10|101829': u'Communication, Création',
        '11|101790': u'.....Communication',
        '12|101791': u'.....Création',
        '13|101792': u'.....Documentation, rédaction technique',
        '14|101793': u'.....Journalisme, édition',
        '15|101830': u'Direction d\'entreprise',
        '16|101794': u'.....Adjoint, conseil de direction',
        '17|101795': u'.....Direction générale',
        '18|101831': u'Etudes, Recherche et Développement',
        '19|101796': u'.....Conception, recherche',
        '20|101797': u'.....Direction recherche et développement',
        '21|101798': u'.....Etudes socio-économiques',
        '22|101799': u'.....Projets scientifiques et techniques',
        '23|101800': u'.....Test, essai, validation, expertise',
        '24|101832': u'Gestion, Finance, Administration',
        '25|101801': u'.....Administration, gestion, organisation',
        '26|101802': u'.....Comptabilité',
        '27|101803': u'.....Contrôle de gestion, audit',
        '28|101804': u'.....Direction gestion, finance',
        '29|101805': u'.....Droit, fiscalité',
        '30|101806': u'.....Finance, trésorerie',
        '31|101833': u'Informatique',
        '32|101807': u'.....Direction informatique',
        '33|101808': u'.....Exploitation, maintenance informatique',
        '34|101809': u'.....Informatique de gestion',
        '35|101810': u'.....Informatique industrielle',
        '36|101811': u'.....Informatique web, sites et portails Internet',
        '37|101812': u'.....Maîtrise d\'ouvrage et fonctionnel',
        '38|101813': u'.....Système, réseaux, données',
        '39|101834': u'Production Industrielle, Travaux, Chantiers',
        '40|101814': u'.....Cadres de chantier',
        '41|101815': u'.....Cadres de production industrielle',
        '42|101816': u'.....Direction d\'unité industrielle',
        '43|101835': u'Ressources Humaines',
        '44|101817': u'.....Administration des RH',
        '45|101818': u'.....Développement des RH',
        '46|101819': u'.....Direction des ressources humaines',
        '47|101820': u'.....Formation initiale et continue',
        '48|101836': u'Sanitaire, Social, Culture',
        '49|101821': u'.....Activités sanitaires, sociales et culturelles',
        '50|101837': u'Services Techniques',
        '51|101822': u'.....Achats',
        '52|101823': u'.....Direction des services techniques',
        '53|101824': u'.....Logistique',
        '54|101825': u'.....Maintenance, sécurité',
        '55|101826': u'.....Process, méthodes',
        '56|101827': u'.....Qualité',
    }.iteritems())])

    secteur_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101752': u'Activités des organisations associatives et administration publique',
        '101753': u'Activités informatiques',
        '101754': u'Activités juridiques et comptables',
        '101755': u'Agroalimentaire',
        '101756': u'Automobile, aéronautique et autres matériels de transport',
        '101757': u'Banque et Assurances',
        '101758': u'Bois - Papier - Imprimerie',
        '101759': u'Chimie - Caoutchouc - Plastique',
        '101760': u'Commerce interentreprises',
        '101761': u'Communication et médias',
        '101762': u'Conseil et gestion des entreprises',
        '101763': u'Construction',
        '101764': u'Distribution généraliste et spécialisée',
        '101765': u'Energies - Eau',
        '101766': u'Equipements électriques et électroniques',
        '101767': u'Formation initiale et continue',
        '101768': u'Gestion des déchets',
        '101769': u'Hôtellerie - Restauration - Loisirs',
        '101770': u'Immobilier',
        '101771': u'Industrie pharmaceutique',
        '101772': u'Ingénierie - R et D',
        '101773': u'Intermédiaires du recrutement',
        '101774': u'Mécanique - Métallurgie',
        '101775': u'Meuble, Textile et autres industries manufacturières',
        '101776': u'Santé - action sociale',
        '101777': u'Services divers aux entreprises',
        '101778': u'Télécommunications',
        '101779': u'Transports et logistique',
    }.iteritems())])

    type_contrat_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101888': u'CDI',
        '101887': u'CDD',
        '101889': u'Travail Temporaire',
    }.iteritems())])

    salary_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101839': u'Moins de 35 K€',
        '101840': u'Entre 35 et 49 K€',
        '101841': u'Entre 50 et 69 K€',
        '101842': u'Entre 70 et 90 K€',
        '101843': u'Plus de 90 K€',
    }.iteritems())])

    date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101850': u'Aujourd\'hui',
        '101851': u'Les 7 derniers jours',
        '101852': u'Les 30 derniers jours',
        '101853': u'Toutes les offres',
    }.iteritems())])

    level_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
        ' ': u'-- Indifférent --',
        '101846': u'Débutant',
        '101848': u'Expérimenté',
    }.iteritems())])

    CONFIG = BackendConfig(Value('place', label=u'Lieu', choices=places_choices, default=''),
                           Value('fonction', label=u'Fonction', choices=fonction_choices, default=''),
                           Value('secteur', label=u'Secteur', choices=secteur_choices, default=''),
                           Value('contrat', label=u'Contrat', choices=type_contrat_choices, default=''),
                           Value('salaire', label=u'Salaire', choices=salary_choices, default=''),
                           Value('limit_date', label=u'Date', choices=date_choices, default=''),
                           Value('level', label=u'Expérience', choices=level_choices, default=''))

    def search_job(self, pattern=None):
        with self.browser:
            for job_advert in self.browser.search_job(pattern=pattern):
                yield job_advert

    def decode_choice(self, choice):
        return choice.split('|')[1]

    def advanced_search_job(self):
        for job_advert in self.browser.advanced_search_job(region=self.decode_choice(self.config['place'].get()),
                                                           fonction=self.decode_choice(self.config['fonction'].get()),
                                                           secteur=self.config['secteur'].get(),
                                                           salaire=self.config['salaire'].get(),
                                                           contrat=self.config['contrat'].get(),
                                                           limit_date=self.config['limit_date'].get(),
                                                           level=self.config['level'].get()):
            yield job_advert

    def get_job_advert(self, _id, advert=None):
        with self.browser:
            return self.browser.get_job_advert(_id, advert)

    def fill_obj(self, advert, fields):
        self.get_job_advert(advert.id, advert)

    OBJECTS = {ApecJobAdvert: fill_obj}
<|endoftext|>


<commit_before>import os
import pickle

from data import data

_DATA = {}
_HISTORY = []
_PICKLE = set()
_PICKLE_PATH = data.project_path('data/')

def register(path, value, pickle=False):
  if path in _DATA:
    raise KeyError('%s already specified' % path)
  _DATA[path] = value
  if pickle:
    _PICKLE.add(path)

def get(path):
  value = _DATA[path]
  if callable(value):
    if path in _PICKLE:
      pickle_src = os.path.join(_PICKLE_PATH, path.replace('/', '_') + '.pkl')
      if os.path.exists(pickle_src):
        # Read from pkl.
        _DATA[path] = pickle.load(_open_pkl_path(pickle_src, 'rb'))
      else:
        # Save to pkl.
        _DATA[path] = value()
        try:
          pickle.dump(_DATA[path], _open_pkl_path(pickle_src, 'wb'))
        except Exception as e:
          print(e)
          raise e
    else:
      _DATA[path] = value()
  return _DATA[path]

def _open_pkl_path(path, mode):
  return open(path, mode)

def init():
  if _DATA:
    raise Exception('Already initialized')
  _DATA['initialized'] = True

def reset():
  _DATA.clear()
  _PICKLE.clear()

def save():
  global _DATA
  _HISTORY.append(_DATA)
  _DATA = {}

def restore():
  global _DATA
  _DATA = _HISTORY.pop()
<commit_msg>Add a "base" warehouse corpus for letter frequencies.<commit_after>import collections
import os
import pickle

from data import data

_DATA = {}
_HISTORY = []
_PICKLE = set()
_PICKLE_PATH = data.project_path('data/')

def register(path, value, pickle=False):
  if path in _DATA:
    raise KeyError('%s already specified' % path)
  _DATA[path] = value
  if pickle:
    _PICKLE.add(path)

def get(path):
  value = _DATA[path]
  if callable(value):
    if path in _PICKLE:
      pickle_src = os.path.join(_PICKLE_PATH, path.replace('/', '_') + '.pkl')
      if os.path.exists(pickle_src):
        # Read from pkl.
        _DATA[path] = pickle.load(_open_pkl_path(pickle_src, 'rb'))
      else:
        # Save to pkl.
        _DATA[path] = value()
        try:
          pickle.dump(_DATA[path], _open_pkl_path(pickle_src, 'wb'))
        except Exception as e:
          print(e)
          raise e
    else:
      _DATA[path] = value()
  return _DATA[path]

def _open_pkl_path(path, mode):
  return open(path, mode)

def init(register_base=True):
  if _DATA:
    raise Exception('Already initialized')
  _DATA['initialized'] = True
  if register_base:
    register('/letter/frequency', _get_letter_frequency)

def reset():
  _DATA.clear()
  _PICKLE.clear()

def save():
  global _DATA
  _HISTORY.append(_DATA)
  _DATA = {}

def restore():
  global _DATA
  _DATA = _HISTORY.pop()

def _get_letter_frequency():
  return collections.OrderedDict(sorted(list(zip('abcdefghijklmnopqrstuvwxyz', [
    # abcdef.
    9081174698, 419765694, 596623239, 361493758, 593086170, 297285731,
    # ghijkl.
    227771642, 220523502, 3086225277, 180739802, 195647953, 252900442,
    341583838, 437961375, 246429812, 303249898, 139563801, 323534251,  # mnopqr.
    565123981, 388448018, 179615587, 204486977, 252231566, 508609523,  # stuvwx.
    195011703, 132095202,  # yz.
  ])), key=lambda x: x[1], reverse=True))
<|endoftext|>