<commit_before>from __future__ import unicode_literals

import os
import sys
import mmap
import shutil
import hashlib
import tempfile
import importlib
import contextlib
import subprocess
import unicodedata

from six import PY3, string_types, binary_type, text_type, iteritems

try:
    import pathlib2 as pathlib
except ImportError:  # pragma: no cover
    import pathlib

from clldutils.misc import UnicodeMixin

Path = pathlib.Path

@contextlib.contextmanager
def sys_path(p):
    p = Path(p).as_posix()
    sys.path.insert(0, p)
    yield
    sys.path.pop(0)

@contextlib.contextmanager
def memorymapped(filename, access=mmap.ACCESS_READ):
    fd = open(as_posix(filename))
    try:
        m = mmap.mmap(fd.fileno(), 0, access=access)
        try:
            yield m
        finally:
            m.close()
    finally:
        fd.close()

def import_module(p):
    with sys_path(p.parent):
        m = importlib.import_module(p.stem)
        if Path(m.__file__).parent not in [p.parent, p]:
            # If we end up importing from the wrong place, raise an error:
            raise ImportError(m.__file__)  # pragma: no cover
        return m

# In python 3, pathlib treats path components and string-like representations or
# attributes of paths (like name and stem) as unicode strings. Unfortunately this is not
# true for pathlib under python 2.7. So as workaround for the case of using non-ASCII
# path names with python 2.7 the following two wrapper functions are provided.
# Note that the issue is even more complex, because pathlib with python 2.7 under windows
# may still pose problems.
def path_component(s, encoding='utf-8'):
    if isinstance(s, binary_type) and PY3:  # pragma: no cover
        s = s.decode(encoding)
    if isinstance(s, text_type) and not PY3:
        s = s.encode(encoding)
    return s

def as_unicode(p, encoding='utf-8'):
    if PY3:  # pragma: no cover
        return '%s' % p
    return (b'%s' % p).decode(encoding)

def as_posix(p):
    if isinstance(p, Path):
        return p.as_posix()
    elif isinstance(p, string_types):
        return Path(p).as_posix()
    raise ValueError(p)

def remove(p):
    os.remove(as_posix(p))

def read_text(p, encoding='utf8', **kw):
    with Path(p).open(encoding=encoding, **kw) as fp:
        return fp.read()

def write_text(p, text, encoding='utf8', **kw):
    with Path(p).open('w', encoding=encoding, **kw) as fp:
        return fp.write(text)

def readlines(p,
              encoding=None,
              strip=False,
              comment=None,
              normalize=None,
              linenumbers=False):
    """
    Read a `list` of lines from a text file.

    :param p: File path (or `list` or `tuple` of text)
    :param encoding: Registered codec.
    :param strip: If `True`, strip leading and trailing whitespace.
    :param comment: String used as syntax to mark comment lines. When not `None`, \
    commented lines will be stripped. This implies `strip=True`.
    :param normalize: 'NFC', 'NFKC', 'NFD', 'NFKD'
    :param linenumbers: return also line numbers.
    :return: `list` of text lines or pairs (`int`, text or `None`).
    """
    if comment:
        strip = True
    if isinstance(p, (list, tuple)):
        res = [l.decode(encoding) if encoding else l for l in p]
    else:
        with Path(p).open(encoding=encoding or 'utf-8') as fp:
            res = fp.readlines()
    if strip:
        res = [l.strip() or None for l in res]
    if comment:
        res = [None if l and l.startswith(comment) else l for l in res]
    if normalize:
        res = [unicodedata.normalize(normalize, l) if l else l for l in res]
    if linenumbers:
        return [(n, l) for n, l in enumerate(res, 1)]
    return [l for l in res if l is not None]

def rmtree(p, **kw):
    return shutil.rmtree(as_posix(p), **kw)

def move(src, dst):
    return shutil.move(as_posix(src), as_posix(dst))

def copy(src, dst):
    return shutil.copy(as_posix(src), as_posix(dst))

def copytree(src, dst, **kw):
    return shutil.copytree(as_posix(src), as_posix(dst), **kw)

def walk(p, mode='all', **kw):
    """Wrapper for `os.walk`, yielding `Path` objects.

    :param p: root of the directory tree to walk.
    :param mode: 'all|dirs|files', defaulting to 'all'.
    :param kw: Keyword arguments are passed to `os.walk`.
    :return: Generator for the requested Path objects.
    """
    for dirpath, dirnames, filenames in os.walk(as_posix(p), **kw):
        if mode in ('all', 'dirs'):
            for dirname in dirnames:
                yield Path(dirpath).joinpath(dirname)
        if mode in ('all', 'files'):
            for fname in filenames:
                yield Path(dirpath).joinpath(fname)

def md5(p, bufsize=32768):
    hash_md5 = hashlib.md5()
    with open(Path(p).as_posix(), 'rb') as fp:
        for chunk in iter(lambda: fp.read(bufsize), b''):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

class Manifest(dict, UnicodeMixin):
    """A `dict` mapping relative path names to md5 sums of file contents.

    A `Manifest.from_dir(d, relative_to=d.parent).__unicode__()` is equivalent
    to the content of the file `manifest-md5.txt` of the BagIt specification.

    .. seealso:: https://en.wikipedia.org/wiki/BagIt
    """

    @classmethod
    def from_dir(cls, d, relative_to=None):
        d = Path(d)
        assert d.is_dir()
        return cls(
            (p.relative_to(relative_to or d).as_posix(), md5(p))
            for p in walk(d, mode='files'))

    def __unicode__(self):
        return '\n'.join('{0}  {1}'.format(v, k)
                         for k, v in sorted(iteritems(self)))

    def write(self, outdir=None):
        path = Path(outdir or '.') / 'manifest-md5.txt'
        write_text(path, '{0}'.format(self))

def git_describe(dir_):
    dir_ = Path(dir_)
    if not dir_.exists():
        raise ValueError('cannot describe non-existent directory')
    dir_ = dir_.resolve()
    cmd = [
        'git', '--git-dir=%s' % dir_.joinpath('.git').as_posix(), 'describe', '--always']
    try:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        if p.returncode == 0:
            res = stdout.strip()  # pragma: no cover
        else:
            raise ValueError(stderr)
    except ValueError:
        res = dir_.name
    if not isinstance(res, text_type):
        res = res.decode('utf8')
    return res

class TemporaryDirectory(object):
    """A trimmed down backport of python 3's tempfile.TemporaryDirectory."""

    def __init__(self, **kw):
        self.name = Path(tempfile.mkdtemp(**kw))

    def __enter__(self):
        return self.name

    def __exit__(self, exc_type, exc_val, exc_tb):
        rmtree(self.name)
<commit_msg>add --tags flag to git_describe call (include lightweight tags)<commit_after>from __future__ import unicode_literals

import os
import sys
import mmap
import shutil
import hashlib
import tempfile
import importlib
import contextlib
import subprocess
import unicodedata

from six import PY3, string_types, binary_type, text_type, iteritems

try:
    import pathlib2 as pathlib
except ImportError:  # pragma: no cover
    import pathlib

from clldutils.misc import UnicodeMixin

Path = pathlib.Path

@contextlib.contextmanager
def sys_path(p):
    p = Path(p).as_posix()
    sys.path.insert(0, p)
    yield
    sys.path.pop(0)

@contextlib.contextmanager
def memorymapped(filename, access=mmap.ACCESS_READ):
    fd = open(as_posix(filename))
    try:
        m = mmap.mmap(fd.fileno(), 0, access=access)
        try:
            yield m
        finally:
            m.close()
    finally:
        fd.close()

def import_module(p):
    with sys_path(p.parent):
        m = importlib.import_module(p.stem)
        if Path(m.__file__).parent not in [p.parent, p]:
            # If we end up importing from the wrong place, raise an error:
            raise ImportError(m.__file__)  # pragma: no cover
        return m

# In python 3, pathlib treats path components and string-like representations or
# attributes of paths (like name and stem) as unicode strings. Unfortunately this is not
# true for pathlib under python 2.7. So as workaround for the case of using non-ASCII
# path names with python 2.7 the following two wrapper functions are provided.
# Note that the issue is even more complex, because pathlib with python 2.7 under windows
# may still pose problems.
def path_component(s, encoding='utf-8'):
    if isinstance(s, binary_type) and PY3:  # pragma: no cover
        s = s.decode(encoding)
    if isinstance(s, text_type) and not PY3:
        s = s.encode(encoding)
    return s

def as_unicode(p, encoding='utf-8'):
    if PY3:  # pragma: no cover
        return '%s' % p
    return (b'%s' % p).decode(encoding)

def as_posix(p):
    if isinstance(p, Path):
        return p.as_posix()
    elif isinstance(p, string_types):
        return Path(p).as_posix()
    raise ValueError(p)

def remove(p):
    os.remove(as_posix(p))

def read_text(p, encoding='utf8', **kw):
    with Path(p).open(encoding=encoding, **kw) as fp:
        return fp.read()

def write_text(p, text, encoding='utf8', **kw):
    with Path(p).open('w', encoding=encoding, **kw) as fp:
        return fp.write(text)

def readlines(p,
              encoding=None,
              strip=False,
              comment=None,
              normalize=None,
              linenumbers=False):
    """
    Read a `list` of lines from a text file.

    :param p: File path (or `list` or `tuple` of text)
    :param encoding: Registered codec.
    :param strip: If `True`, strip leading and trailing whitespace.
    :param comment: String used as syntax to mark comment lines. When not `None`, \
    commented lines will be stripped. This implies `strip=True`.
    :param normalize: 'NFC', 'NFKC', 'NFD', 'NFKD'
    :param linenumbers: return also line numbers.
    :return: `list` of text lines or pairs (`int`, text or `None`).
    """
    if comment:
        strip = True
    if isinstance(p, (list, tuple)):
        res = [l.decode(encoding) if encoding else l for l in p]
    else:
        with Path(p).open(encoding=encoding or 'utf-8') as fp:
            res = fp.readlines()
    if strip:
        res = [l.strip() or None for l in res]
    if comment:
        res = [None if l and l.startswith(comment) else l for l in res]
    if normalize:
        res = [unicodedata.normalize(normalize, l) if l else l for l in res]
    if linenumbers:
        return [(n, l) for n, l in enumerate(res, 1)]
    return [l for l in res if l is not None]

def rmtree(p, **kw):
    return shutil.rmtree(as_posix(p), **kw)

def move(src, dst):
    return shutil.move(as_posix(src), as_posix(dst))

def copy(src, dst):
    return shutil.copy(as_posix(src), as_posix(dst))

def copytree(src, dst, **kw):
    return shutil.copytree(as_posix(src), as_posix(dst), **kw)

def walk(p, mode='all', **kw):
    """Wrapper for `os.walk`, yielding `Path` objects.

    :param p: root of the directory tree to walk.
    :param mode: 'all|dirs|files', defaulting to 'all'.
    :param kw: Keyword arguments are passed to `os.walk`.
    :return: Generator for the requested Path objects.
    """
    for dirpath, dirnames, filenames in os.walk(as_posix(p), **kw):
        if mode in ('all', 'dirs'):
            for dirname in dirnames:
                yield Path(dirpath).joinpath(dirname)
        if mode in ('all', 'files'):
            for fname in filenames:
                yield Path(dirpath).joinpath(fname)

def md5(p, bufsize=32768):
    hash_md5 = hashlib.md5()
    with open(Path(p).as_posix(), 'rb') as fp:
        for chunk in iter(lambda: fp.read(bufsize), b''):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

class Manifest(dict, UnicodeMixin):
    """A `dict` mapping relative path names to md5 sums of file contents.

    A `Manifest.from_dir(d, relative_to=d.parent).__unicode__()` is equivalent
    to the content of the file `manifest-md5.txt` of the BagIt specification.

    .. seealso:: https://en.wikipedia.org/wiki/BagIt
    """

    @classmethod
    def from_dir(cls, d, relative_to=None):
        d = Path(d)
        assert d.is_dir()
        return cls(
            (p.relative_to(relative_to or d).as_posix(), md5(p))
            for p in walk(d, mode='files'))

    def __unicode__(self):
        return '\n'.join('{0}  {1}'.format(v, k)
                         for k, v in sorted(iteritems(self)))

    def write(self, outdir=None):
        path = Path(outdir or '.') / 'manifest-md5.txt'
        write_text(path, '{0}'.format(self))

def git_describe(dir_):
    dir_ = Path(dir_)
    if not dir_.exists():
        raise ValueError('cannot describe non-existent directory')
    dir_ = dir_.resolve()
    cmd = [
        'git', '--git-dir=%s' % dir_.joinpath('.git').as_posix(), 'describe',
        '--always', '--tags']
    try:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        if p.returncode == 0:
            res = stdout.strip()  # pragma: no cover
        else:
            raise ValueError(stderr)
    except ValueError:
        res = dir_.name
    if not isinstance(res, text_type):
        res = res.decode('utf8')
    return res

class TemporaryDirectory(object):
    """A trimmed down backport of python 3's tempfile.TemporaryDirectory."""

    def __init__(self, **kw):
        self.name = Path(tempfile.mkdtemp(**kw))

    def __enter__(self):
        return self.name

    def __exit__(self, exc_type, exc_val, exc_tb):
        rmtree(self.name)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import numpy as np
import os
from PIL import Image
from collections import defaultdict
from itertools import product

CNN_IN_WIDTH = 64
CNN_IN_HEIGHT = 32

DATA_AUG_POS_SHIFT_MIN = -2
DATA_AUG_POS_SHIFT_MAX = 2
DATA_AUG_SCALES = [0.9, 1.1]
DATA_AUG_ROT_MIN = -15
DATA_AUG_ROT_MAX = 15

TRAIN_DIR = 'flickr_logos_27_dataset'
TRAIN_IMAGE_DIR = os.path.join(TRAIN_DIR, 'flickr_logos_27_dataset_images')
CROPPED_IMAGE_DIR = os.path.join(TRAIN_DIR,
                                 'flickr_logos_27_dataset_cropped_images')

def parse_annot(annot):
    fn = annot[0].decode('utf-8')
    class_name = annot[1].decode('utf-8')
    train_subset_class = annot[2].decode('utf-8')
    return fn, class_name, train_subset_class

def get_rect(annot):
    rect = defaultdict(int)
    x1, y1, x2, y2 = rect_coord(annot[3:])
    cx, cy, wid, hgt = center_wid_hgt(x1, y1, x2, y2)
    rect['x1'] = x1
    rect['y1'] = y1
    rect['x2'] = x2
    rect['y2'] = y2
    rect['cx'] = cx
    rect['cy'] = cy
    rect['wid'] = wid
    rect['hgt'] = hgt
    return rect

def aug_pos(annot, im):
    aug_pos_ims = []
    aug_pos_suffixes = []

    rect = get_rect(annot)
    for sx, sy in product(range(DATA_AUG_POS_SHIFT_MIN, DATA_AUG_POS_SHIFT_MAX), range(DATA_AUG_POS_SHIFT_MIN, DATA_AUG_POS_SHIFT_MAX)):
        cx = rect['cx'] + sx
        cy = rect['cy'] + sy
        cropped_im = im.crop((cx - rect['wid'] // 2, cy - rect['hgt'] // 2,
                              cx + rect['wid'] // 2, cy + rect['hgt'] // 2))
        resized_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
        aug_pos_ims.append(resized_im)
        aug_pos_suffixes.append('p' + str(cx) + str(cy))
        cropped_im.close()

    return aug_pos_ims, aug_pos_suffixes

def aug_scale(annot, im):
    aug_scale_ims = []
    aug_scale_suffixes = []

    rect = get_rect(annot)
    for s in DATA_AUG_SCALES:
        w = int(rect['wid'] * s)
        h = int(rect['hgt'] * s)
        cropped_im = im.crop((rect['cx'] - w // 2, rect['cy'] - h // 2,
                              rect['cx'] + w // 2, rect['cy'] + h // 2))
        resized_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
        aug_scale_ims.append(resized_im)
        aug_scale_suffixes.append('s' + str(s))
        cropped_im.close()

    return aug_scale_ims, aug_scale_suffixes

def aug_rot(annot, im):
    aug_rot_ims = []
    aug_rot_suffixes = []

    rect = get_rect(annot)
    for r in range(DATA_AUG_ROT_MIN, DATA_AUG_ROT_MAX):
        rotated_im = im.rotate(r)
        cropped_im = rotated_im.crop((rect['cx'] - rect['wid'] // 2, rect['cy'] - rect['hgt'] // 2,
                                      rect['cx'] + rect['wid'] // 2, rect['cy'] + rect['hgt'] // 2))
        resized_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
        aug_rot_ims.append(resized_im)
        aug_rot_suffixes.append('r' + str(r))
        rotated_im.close()
        cropped_im.close()

    return aug_rot_ims, aug_rot_suffixes

def crop_logos(annot, im):
    x1, y1, x2, y2 = rect_coord(annot[3:])
    cropped_im = im.crop((x1, y1, x2, y2))
    cropped_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
    cropped_suffix = 'p00'
    return cropped_im, cropped_suffix

def rect_coord(annot_part):
    return list(map(int, annot_part))

def center_wid_hgt(x1, y1, x2, y2):
    cx = x1 + (x2 - x1) // 2
    cy = y1 + (y2 - y1) // 2
    wid = (x2 - x1)
    hgt = (y2 - y1)
    return cx, cy, wid, hgt

def is_skip(annot_part):
    x1, y1, x2, y2 = rect_coord(annot_part)
    _, _, wid, hgt = center_wid_hgt(x1, y1, x2, y2)
    if wid <= 0 or hgt <= 0:
        return True
    else:
        return False

def save_im(annot, cnt, *args):
    fn, class_name, train_subset_class = parse_annot(annot)
    for arg in args:
        im, suffix = arg
        save_fn = '_'.join([fn.split('.')[0], class_name,
                            train_subset_class, str(cnt), suffix]) + os.path.splitext(fn)[1]
        im.save(os.path.join(CROPPED_IMAGE_DIR, save_fn))

def close_im(*args):
    for im in args:
        im.close()

def crop_and_aug(annot_train):
    if not os.path.exists(CROPPED_IMAGE_DIR):
        os.makedirs(CROPPED_IMAGE_DIR)

    cnt_per_file = defaultdict(int)
    for annot in annot_train:
        # for generating a file name
        fn, _, _ = parse_annot(annot)
        cnt_per_file[fn] += 1

        # skip if width or height equal zero
        if is_skip(annot[3:]):
            print('Skip: ', fn)
            continue

        # open an image
        im = Image.open(os.path.join(TRAIN_IMAGE_DIR, fn))

        # normal cropping
        cropped_im, cropped_suffix = crop_logos(annot, im)

        # augment by shifting a center
        shifted_ims, shifted_suffixes = aug_pos(annot, im)

        # augment by scaling
        scaled_ims, scaled_suffixes = aug_scale(annot, im)

        # augment by rotation
        rotated_im, rotated_suffix = aug_rot(annot, im)

        # save images
        save_im(annot, cnt_per_file[fn],
                [cropped_im, cropped_suffix],
                [shifted_ims, shifted_suffixes],
                [scaled_ims, scaled_suffixes],
                [rotated_ims, rotated_suffixes])

        # close image file
        close_im(im, cropped_im, shifted_ims, scaled_ims, rotated_ims)

    # print results
    org_imgs = [img for img in os.listdir(TRAIN_IMAGE_DIR)]
    crop_and_aug_imgs = [img for img in os.listdir(CROPPED_IMAGE_DIR)]
    print('original: %d' % (len(org_imgs)))
    print('cropped: %d' % (len(crop_and_aug_imgs)))

    return crop_and_aug_imgs

def main():
    annot_train = np.loadtxt(
        os.path.join(TRAIN_DIR,
                     'flickr_logos_27_dataset_training_set_annotation.txt'),
        dtype='a')
    print('train_annotation: %d, %d ' % (annot_train.shape))

    # normal cropping and data augmentation
    crop_and_aug(annot_train)

if __name__ == '__main__':
    main()
<commit_msg>Fix suffixes<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import numpy as np
import os
from PIL import Image
from collections import defaultdict
from itertools import product

CNN_IN_WIDTH = 64
CNN_IN_HEIGHT = 32

DATA_AUG_POS_SHIFT_MIN = -2
DATA_AUG_POS_SHIFT_MAX = 2
DATA_AUG_SCALES = [0.9, 1.1]
DATA_AUG_ROT_MIN = -15
DATA_AUG_ROT_MAX = 15

TRAIN_DIR = 'flickr_logos_27_dataset'
TRAIN_IMAGE_DIR = os.path.join(TRAIN_DIR, 'flickr_logos_27_dataset_images')
CROPPED_IMAGE_DIR = os.path.join(TRAIN_DIR,
                                 'flickr_logos_27_dataset_cropped_images')

def parse_annot(annot):
    fn = annot[0].decode('utf-8')
    class_name = annot[1].decode('utf-8')
    train_subset_class = annot[2].decode('utf-8')
    return fn, class_name, train_subset_class

def get_rect(annot):
    rect = defaultdict(int)
    x1, y1, x2, y2 = rect_coord(annot[3:])
    cx, cy, wid, hgt = center_wid_hgt(x1, y1, x2, y2)
    rect['x1'] = x1
    rect['y1'] = y1
    rect['x2'] = x2
    rect['y2'] = y2
    rect['cx'] = cx
    rect['cy'] = cy
    rect['wid'] = wid
    rect['hgt'] = hgt
    return rect

def aug_pos(annot, im):
    aug_pos_ims = []
    aug_pos_suffixes = []

    rect = get_rect(annot)
    for sx, sy in product(range(DATA_AUG_POS_SHIFT_MIN, DATA_AUG_POS_SHIFT_MAX), range(DATA_AUG_POS_SHIFT_MIN, DATA_AUG_POS_SHIFT_MAX)):
        cx = rect['cx'] + sx
        cy = rect['cy'] + sy
        cropped_im = im.crop((cx - rect['wid'] // 2, cy - rect['hgt'] // 2,
                              cx + rect['wid'] // 2, cy + rect['hgt'] // 2))
        resized_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
        aug_pos_ims.append(resized_im)
        aug_pos_suffixes.append('p' +  str(sx) + str(sy))
        cropped_im.close()

    return aug_pos_ims, aug_pos_suffixes

def aug_scale(annot, im):
    aug_scale_ims = []
    aug_scale_suffixes = []

    rect = get_rect(annot)
    for s in DATA_AUG_SCALES:
        w = int(rect['wid'] * s)
        h = int(rect['hgt'] * s)
        cropped_im = im.crop((rect['cx'] - w // 2, rect['cy'] - h // 2,
                              rect['cx'] + w // 2, rect['cy'] + h // 2))
        resized_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
        aug_scale_ims.append(resized_im)
        aug_scale_suffixes.append('s' + str(s))
        cropped_im.close()

    return aug_scale_ims, aug_scale_suffixes

def aug_rot(annot, im):
    aug_rot_ims = []
    aug_rot_suffixes = []

    rect = get_rect(annot)
    for r in range(DATA_AUG_ROT_MIN, DATA_AUG_ROT_MAX):
        rotated_im = im.rotate(r)
        cropped_im = rotated_im.crop((rect['cx'] - rect['wid'] // 2, rect['cy'] - rect['hgt'] // 2,
                                      rect['cx'] + rect['wid'] // 2, rect['cy'] + rect['hgt'] // 2))
        resized_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
        aug_rot_ims.append(resized_im)
        aug_rot_suffixes.append('r' + str(r))
        rotated_im.close()
        cropped_im.close()

    return aug_rot_ims, aug_rot_suffixes

def crop_logos(annot, im):
    x1, y1, x2, y2 = rect_coord(annot[3:])
    cropped_im = im.crop((x1, y1, x2, y2))
    cropped_im = cropped_im.resize((CNN_IN_WIDTH, CNN_IN_HEIGHT))
    cropped_suffix = 'p00'
    return cropped_im, cropped_suffix

def rect_coord(annot_part):
    return list(map(int, annot_part))

def center_wid_hgt(x1, y1, x2, y2):
    cx = x1 + (x2 - x1) // 2
    cy = y1 + (y2 - y1) // 2
    wid = (x2 - x1)
    hgt = (y2 - y1)
    return cx, cy, wid, hgt

def is_skip(annot_part):
    x1, y1, x2, y2 = rect_coord(annot_part)
    _, _, wid, hgt = center_wid_hgt(x1, y1, x2, y2)
    if wid <= 0 or hgt <= 0:
        return True
    else:
        return False

def save_im(annot, cnt, *args):
    fn, class_name, train_subset_class = parse_annot(annot)
    for arg in args:
        im, suffix = arg
        save_fn = '_'.join([fn.split('.')[0], class_name,
                            train_subset_class, str(cnt), suffix]) + os.path.splitext(fn)[1]
        im.save(os.path.join(CROPPED_IMAGE_DIR, save_fn))

def close_im(*args):
    for im in args:
        im.close()

def crop_and_aug(annot_train):
    if not os.path.exists(CROPPED_IMAGE_DIR):
        os.makedirs(CROPPED_IMAGE_DIR)

    cnt_per_file = defaultdict(int)
    for annot in annot_train:
        # for generating a file name
        fn, _, _ = parse_annot(annot)
        cnt_per_file[fn] += 1

        # skip if width or height equal zero
        if is_skip(annot[3:]):
            print('Skip: ', fn)
            continue

        # open an image
        im = Image.open(os.path.join(TRAIN_IMAGE_DIR, fn))

        # normal cropping
        cropped_im, cropped_suffix = crop_logos(annot, im)

        # augment by shifting a center
        shifted_ims, shifted_suffixes = aug_pos(annot, im)

        # augment by scaling
        scaled_ims, scaled_suffixes = aug_scale(annot, im)

        # augment by rotation
        rotated_im, rotated_suffix = aug_rot(annot, im)

        # save images
        save_im(annot, cnt_per_file[fn],
                [cropped_im, cropped_suffix],
                [shifted_ims, shifted_suffixes],
                [scaled_ims, scaled_suffixes],
                [rotated_ims, rotated_suffixes])

        # close image file
        close_im(im, cropped_im, shifted_ims, scaled_ims, rotated_ims)

    # print results
    org_imgs = [img for img in os.listdir(TRAIN_IMAGE_DIR)]
    crop_and_aug_imgs = [img for img in os.listdir(CROPPED_IMAGE_DIR)]
    print('original: %d' % (len(org_imgs)))
    print('cropped: %d' % (len(crop_and_aug_imgs)))

    return crop_and_aug_imgs

def main():
    annot_train = np.loadtxt(
        os.path.join(TRAIN_DIR,
                     'flickr_logos_27_dataset_training_set_annotation.txt'),
        dtype='a')
    print('train_annotation: %d, %d ' % (annot_train.shape))

    # normal cropping and data augmentation
    crop_and_aug(annot_train)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from datetime import datetime

from django import forms
from django.core.paginator import Paginator
from django.db import models
from django.template.context import RequestContext
from django.template.loader import render_to_string
from django.utils.datastructures import SortedDict
from django.utils.translation import ugettext_lazy as _

from .models import Event

class EventsContent(models.Model):
    filter = models.CharField(max_length=1, choices=(
                                ('a',_('all')),
                                ('u',_('upcoming')),
                                ('p',_('past')),
                            ))

    class Meta:
        abstract = True
        verbose_name = _('event list')
        verbose_name_plural = _('event lists')

    @property
    def media(self):
        media = forms.Media()
        # TODO really? Fix paths
        media.add_js(('/media/sys/feinheit/js/jquery.scrollTo-min.js',
                      'lib/fancybox/jquery.fancybox-1.3.1.pack.js'))
        media.add_css({'all': ('lib/fancybox/jquery.fancybox-1.3.1.css', )})

        return media

    def render(self, request, context, **kwargs):
        if self.filter == 'u':
            object_list = Event.objects.upcoming()
        elif self.filter == 'p':
            object_list = Event.objects.past()
        else:
            object_list = Event.objects.active()

        # TODO InvalidPage exceptions etc.
        current_page = request.GET.get('page', 1)
        page = Paginator(object_list, 20).page(current_page)

        return render_to_string('content/agenda/event_list.html', {
            'object_list': object_list,
            'page': page,
            }, context_instance=context)

class EventMapContent(models.Model):
    filter = models.CharField(max_length=1, choices=(
                                ('a',_('all')),
                                ('u',_('upcoming')),
                                ('p',_('past')),
                            ))

    class Meta:
        abstract = True
        verbose_name = _('event map')
        verbose_name = _('event maps')

    @property
    def media(self):
        media = forms.Media()
        media.add_js(('http://maps.google.com/maps/api/js?sensor=false',
                      '/media/js/event_map.js'))
        media.add_css({'all': ('/media/css/event_map.css', )})
        return media

    def render(self, request, context, **kwargs):
        return render_to_string('content/agenda/event_map.html',
            context_instance=context)

class EventDateFilterContent(models.Model):
    class Meta:
        abstract = True
        verbose_name = _('Event date filter content')
        verbose_name_plural = _('Event date filter contents')

    def render(self, **kwargs):
        dates_dict = SortedDict()
        dates = Event.objects.active().dates('start_date', 'day', order="DESC")

        for date in dates:
            # TODO unstupidify this code using setdefault etc.
            if not date.year in dates_dict:
                dates_dict[date.year] = SortedDict([
                    ('date', datetime(date.year, 1, 1)),
                    ('months', SortedDict()),
                    ])

            if not date.month in dates_dict[date.year]['months']:
                dates_dict[date.year]['months'][date.month] = SortedDict([
                    ('date', datetime(date.year, date.month, 1)),
                    ('days', SortedDict()),
                    ])

            if not date.day in dates_dict[date.year]['months'][date.month]['days']:
                dates_dict[date.year]['months'][date.month]['days'][date.day] = SortedDict([
                    ('date': datetime(date.year, date.month, date.day)),
                    ])

        return render_to_string('content/agenda/date_filter.html', {
            'content': self,
            'dates_dict': dates_dict,
            }, context_instance=kwargs.get('context'))
<commit_msg>Fix a typo<commit_after>from datetime import datetime

from django import forms
from django.core.paginator import Paginator
from django.db import models
from django.template.context import RequestContext
from django.template.loader import render_to_string
from django.utils.datastructures import SortedDict
from django.utils.translation import ugettext_lazy as _

from .models import Event

class EventsContent(models.Model):
    filter = models.CharField(max_length=1, choices=(
                                ('a',_('all')),
                                ('u',_('upcoming')),
                                ('p',_('past')),
                            ))

    class Meta:
        abstract = True
        verbose_name = _('event list')
        verbose_name_plural = _('event lists')

    @property
    def media(self):
        media = forms.Media()
        # TODO really? Fix paths
        media.add_js(('/media/sys/feinheit/js/jquery.scrollTo-min.js',
                      'lib/fancybox/jquery.fancybox-1.3.1.pack.js'))
        media.add_css({'all': ('lib/fancybox/jquery.fancybox-1.3.1.css', )})

        return media

    def render(self, request, context, **kwargs):
        if self.filter == 'u':
            object_list = Event.objects.upcoming()
        elif self.filter == 'p':
            object_list = Event.objects.past()
        else:
            object_list = Event.objects.active()

        # TODO InvalidPage exceptions etc.
        current_page = request.GET.get('page', 1)
        page = Paginator(object_list, 20).page(current_page)

        return render_to_string('content/agenda/event_list.html', {
            'object_list': object_list,
            'page': page,
            }, context_instance=context)

class EventMapContent(models.Model):
    filter = models.CharField(max_length=1, choices=(
                                ('a',_('all')),
                                ('u',_('upcoming')),
                                ('p',_('past')),
                            ))

    class Meta:
        abstract = True
        verbose_name = _('event map')
        verbose_name = _('event maps')

    @property
    def media(self):
        media = forms.Media()
        media.add_js(('http://maps.google.com/maps/api/js?sensor=false',
                      '/media/js/event_map.js'))
        media.add_css({'all': ('/media/css/event_map.css', )})
        return media

    def render(self, request, context, **kwargs):
        return render_to_string('content/agenda/event_map.html',
            context_instance=context)

class EventDateFilterContent(models.Model):
    class Meta:
        abstract = True
        verbose_name = _('Event date filter content')
        verbose_name_plural = _('Event date filter contents')

    def render(self, **kwargs):
        dates_dict = SortedDict()
        dates = Event.objects.active().dates('start_date', 'day', order="DESC")

        for date in dates:
            # TODO unstupidify this code using setdefault etc.
            if not date.year in dates_dict:
                dates_dict[date.year] = SortedDict([
                    ('date', datetime(date.year, 1, 1)),
                    ('months', SortedDict()),
                    ])

            if not date.month in dates_dict[date.year]['months']:
                dates_dict[date.year]['months'][date.month] = SortedDict([
                    ('date', datetime(date.year, date.month, 1)),
                    ('days', SortedDict()),
                    ])

            if not date.day in dates_dict[date.year]['months'][date.month]['days']:
                dates_dict[date.year]['months'][date.month]['days'][date.day] = SortedDict([
                    ('date', datetime(date.year, date.month, date.day)),
                    ])

        return render_to_string('content/agenda/date_filter.html', {
            'content': self,
            'dates_dict': dates_dict,
            }, context_instance=kwargs.get('context'))
<|endoftext|>


<commit_before>import socket

from tornado import ioloop, iostream

delimiter = '\r\n\r\n'

def read_command():
    stream.read_until(delimiter, on_headers)

def on_headers(data):
    data = data.replace('\r\n\r\n', '')
    print '<< ' + data
    tokens = data.split(' ')
    ts = tokens[1]
    resource = tokens[2]
    content = open(resource).read()
    response = 'CONTENT %s %s' % (ts, content)
    print '>> ' + response
    stream.write(response + delimiter)
    read_command()

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream.connect(('localhost', 8890), read_command)
ioloop.IOLoop.instance().start()<commit_msg>auth + command handling in client<commit_after>import socket

from tornado import ioloop, iostream

delimiter = '\r\n\r\n'

def recv_data(callback):
    while stream.reading() == True:
        continue

    def onrecv(data):
        data = data.replace(delimiter, '')
        print '<< ' + data
        callback(data)

    stream.read_until(delimiter, onrecv)

def send_data(data, callback):
    print '>> '  + data
    stream.write(data + delimiter, callback)

def process_commands(data):
    tokens = data.split(' ')
    cmd = tokens[0]
    args = tokens[1:]

    if cmd == 'ACK':
        read_command()

    elif cmd == 'FETCH':
        on_fetch(args)

    else:
        read_command()

def authenticate():
    send_data('IDENT ashish locker', read_command)

def read_command():
    recv_data(process_commands)

def on_fetch(args):
    ts = args[0]
    resource = args[1]
    try:
        content = open(resource).read()
    except:
        content = 'Not found.'

    response = 'CONTENT {0} {1}'.format(ts, content)
    send_data(response, read_command)

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream.connect(('localhost', 8890), authenticate)
ioloop.IOLoop.instance().start()<|endoftext|>


<commit_before><commit_msg>Wrap ES search query in parens to perserve semantics<commit_after><|endoftext|>


<commit_before><commit_msg>Added the correct git commit.<commit_after><|endoftext|>


<commit_before>import io
import os
import sys

from struct import pack, unpack, calcsize
#

import re

numbers = re.compile('([0-9]+)')

import numpy as np
import matplotlib.pyplot as plt
try:
  import seaborn
except ImportError as e:
  print("seaborn is not available!")

import glob

def numerical_match_first(value):
  if len(numbers.findall(os.path.basename(value))) < 1:
    raise ValueError('Value = ' + value + " doesn't contain a number")
  parts = int( numbers.findall(os.path.basename(value))[0] )
  return parts

#I <---> unsigned 32 bit int
struct_format = 'IdI'
struct_format_uint32  = 'I'
struct_format_doses   = 'd'
struct_format_indices = 'I'

y_slice = 70

#this needs to be made general
def calculate_x_indices():
  x_bins = np.linspace(-26, 24, 126)
  low = np.min(np.where(x_bins >= -2 ))
  high = np.max(np.where(x_bins < 7  ))
  print(x_bins[low], x_bins[high])
  return (low, high)

def slice_indices(lower_edge, upper_edge, lower_bound, upper_bound):
  return None

#this needs to be made general
def calculate_z_indices():
  z_bins = np.linspace(-25, 25, 126)
  low = np.min(np.where(z_bins >= -1 ))
  high = np.max(np.where(z_bins < 4  ))
  print(z_bins[low], z_bins[high])
  return (low, high)

def main(show_spectrum=False):
  #so, now have the correct indices to slice out the parameters
  low_x, high_x  = calculate_x_indices()
  low_z, high_z  = calculate_z_indices()

  # Need the size of binary reads
  read_count   = calcsize(struct_format_uint32)
  read_dose    = calcsize(struct_format_doses)
  read_indices = calcsize(struct_format_indices)

  #these files are the files to read
  files = glob.glob('data_files/beamlet*.dat')
  g = sorted(files, key=numerical_match_first)
  print(g)

  #just iterate over the globbed files;
  for dat_file in g:
    print(dat_file)
    number = numerical_match_first(dat_file)
    outfile = "output_" + str(number) + ".dat"
    #need to keep the number extension the same
    print(dat_file, outfile)
    with open(dat_file, 'rb') as f:
      data = f.read(read_count)
      count, = unpack(struct_format_uint32, data)
      print(count)
      list_doses   = []
      list_indices = []
      for i in range(count):
        data = f.read(read_dose)
        s, = unpack(struct_format_doses, data)
        list_doses.append(s)
      #print(s)
      for i in range(count):
        data = f.read(read_indices)
        s, = unpack(struct_format_indices, data)
        list_indices.append(s)
      #print(s)

      dose_grid = np.zeros((125,125,125))
      for dose, index in zip(list_doses, list_indices):
        dose_grid[np.unravel_index(index, dims=(125,125,125))] = dose

      # print("argmax = ", np.argmax(dose_grid), np.unravel_index(np.argmax(dose_grid), dose_grid.shape))
      # plt.figure()
      # dose_max_slice = np.max(dose_grid[:, y_slice, :])
      # plt.imshow(dose_grid[:, y_slice, :] / dose_max_slice, cmap=plt.get_cmap('cool'))
      # ax = plt.gca()
      # ax.grid(False)
      # plt.show()

      sub_dose_grid = np.array(dose_grid[low_x:high_x+1, :, low_z:high_z+1])
      print("shape:      ", sub_dose_grid.shape)
      print("# elements: ", len(np.nonzero(sub_dose_grid.ravel())[0]))
      # print("elements:   ", np.nonzero(sub_dose_grid.ravel()))

      elements = len(np.nonzero(sub_dose_grid.ravel())[0])
      non_zero_index = np.nonzero(sub_dose_grid.ravel())[0]
      sub_dose_grid_ravel = sub_dose_grid.ravel()

      with open(outfile, 'wb') as out_file:
        out_file.write(pack('I', elements))
        for index_value in non_zero_index:
          out_file.write(pack('d', sub_dose_grid_ravel[index_value]))
        for index_value in non_zero_index:
          out_file.write(pack('I', index_value))

      with open('test_file_ascii_1.dat', 'w') as ascii_file:
        ascii_file.write(str(elements) + " ")
        for index_value in non_zero_index:
          ascii_file.write(str(sub_dose_grid_ravel[index_value]) + " ")
        for index_value in non_zero_index:
          ascii_file.write(str(index_value) + " ")

      # with open('test_file_full_ascii_1.dat', 'w') as ascii_file:
      #   ascii_file.write(str(elements) + " ")
      #   for index_value in non_zero_index:
      #     ascii_file.write(str(sub_dose_grid_ravel[index_value]) + " ")
      #   for index_value in non_zero_index:
      #     ascii_file.write(str(index_value) + " ")

      #print(125**3)
      # print("converting to numpy array")
      # doses = np.array(list_doses)
      # print(np.mean(doses))
      # print(np.std(doses))
      # print("done calculation")
      # print(np.unravel_index(list_indices, (125,125,125)))

      if show_spectrum is True:
        plt.figure()
        ax = plt.subplot(111)
        plt.hist(doses, bins=1000)
        ax.set_yscale("log", nonposy='clip')
        plt.show()
  return 0

if __name__ == '__main__':
  main(show_spectrum=False)
<commit_msg>Removing import seaborn; (1) can be added later to make the plots look nice;<commit_after>import io
import os
import sys

from struct import pack, unpack, calcsize
#

import re

numbers = re.compile('([0-9]+)')

import numpy as np
import matplotlib.pyplot as plt
# try:
#   import seaborn
# except ImportError as e:
#   print("seaborn is not available!")

import glob

def numerical_match_first(value):
  if len(numbers.findall(os.path.basename(value))) < 1:
    raise ValueError('Value = ' + value + " doesn't contain a number")
  parts = int( numbers.findall(os.path.basename(value))[0] )
  return parts

#I <---> unsigned 32 bit int
struct_format = 'IdI'
struct_format_uint32  = 'I'
struct_format_doses   = 'd'
struct_format_indices = 'I'

y_slice = 70

#this needs to be made general
def calculate_x_indices():
  x_bins = np.linspace(-26, 24, 126)
  low = np.min(np.where(x_bins >= -2 ))
  high = np.max(np.where(x_bins < 7  ))
  print(x_bins[low], x_bins[high])
  return (low, high)

def slice_indices(lower_edge, upper_edge, lower_bound, upper_bound):
  return None

#this needs to be made general
def calculate_z_indices():
  z_bins = np.linspace(-25, 25, 126)
  low = np.min(np.where(z_bins >= -1 ))
  high = np.max(np.where(z_bins < 4  ))
  print(z_bins[low], z_bins[high])
  return (low, high)

def main(show_spectrum=False):
  #so, now have the correct indices to slice out the parameters
  low_x, high_x  = calculate_x_indices()
  low_z, high_z  = calculate_z_indices()

  # Need the size of binary reads
  read_count   = calcsize(struct_format_uint32)
  read_dose    = calcsize(struct_format_doses)
  read_indices = calcsize(struct_format_indices)

  #these files are the files to read
  files = glob.glob('data_files/beamlet*.dat')
  g = sorted(files, key=numerical_match_first)
  print(g)

  #just iterate over the globbed files;
  for dat_file in g:
    print(dat_file)
    number = numerical_match_first(dat_file)
    outfile = "output_" + str(number) + ".dat"
    #need to keep the number extension the same
    print(dat_file, outfile)
    with open(dat_file, 'rb') as f:
      data = f.read(read_count)
      count, = unpack(struct_format_uint32, data)
      print(count)
      list_doses   = []
      list_indices = []
      for i in range(count):
        data = f.read(read_dose)
        s, = unpack(struct_format_doses, data)
        list_doses.append(s)
      #print(s)
      for i in range(count):
        data = f.read(read_indices)
        s, = unpack(struct_format_indices, data)
        list_indices.append(s)
      #print(s)

      dose_grid = np.zeros((125,125,125))
      for dose, index in zip(list_doses, list_indices):
        dose_grid[np.unravel_index(index, dims=(125,125,125))] = dose

      # print("argmax = ", np.argmax(dose_grid), np.unravel_index(np.argmax(dose_grid), dose_grid.shape))
      # plt.figure()
      # dose_max_slice = np.max(dose_grid[:, y_slice, :])
      # plt.imshow(dose_grid[:, y_slice, :] / dose_max_slice, cmap=plt.get_cmap('cool'))
      # ax = plt.gca()
      # ax.grid(False)
      # plt.show()

      sub_dose_grid = np.array(dose_grid[low_x:high_x+1, :, low_z:high_z+1])
      print("shape:      ", sub_dose_grid.shape)
      print("# elements: ", len(np.nonzero(sub_dose_grid.ravel())[0]))
      # print("elements:   ", np.nonzero(sub_dose_grid.ravel()))

      elements = len(np.nonzero(sub_dose_grid.ravel())[0])
      non_zero_index = np.nonzero(sub_dose_grid.ravel())[0]
      sub_dose_grid_ravel = sub_dose_grid.ravel()

      with open(outfile, 'wb') as out_file:
        out_file.write(pack('I', elements))
        for index_value in non_zero_index:
          out_file.write(pack('d', sub_dose_grid_ravel[index_value]))
        for index_value in non_zero_index:
          out_file.write(pack('I', index_value))

      with open('test_file_ascii_1.dat', 'w') as ascii_file:
        ascii_file.write(str(elements) + " ")
        for index_value in non_zero_index:
          ascii_file.write(str(sub_dose_grid_ravel[index_value]) + " ")
        for index_value in non_zero_index:
          ascii_file.write(str(index_value) + " ")

      # with open('test_file_full_ascii_1.dat', 'w') as ascii_file:
      #   ascii_file.write(str(elements) + " ")
      #   for index_value in non_zero_index:
      #     ascii_file.write(str(sub_dose_grid_ravel[index_value]) + " ")
      #   for index_value in non_zero_index:
      #     ascii_file.write(str(index_value) + " ")

      #print(125**3)
      # print("converting to numpy array")
      # doses = np.array(list_doses)
      # print(np.mean(doses))
      # print(np.std(doses))
      # print("done calculation")
      # print(np.unravel_index(list_indices, (125,125,125)))

      if show_spectrum is True:
        plt.figure()
        ax = plt.subplot(111)
        plt.hist(doses, bins=1000)
        ax.set_yscale("log", nonposy='clip')
        plt.show()
  return 0

if __name__ == '__main__':
  main(show_spectrum=False)
<|endoftext|>


<commit_before>#!/usr/bin/python

from collections import namedtuple
import gzip
import struct
import sys

Bandwidth = namedtuple('Bandwidth', 'date up down')
Date = namedtuple('Date', 'year month day')

class RStatsHistory(object):
    ID_V0 = 0x30305352
    ID_V1 = 0x31305352

    def __init__(self, rstats_filename):
        self.daily = []
        self.monthly = []
        self.rstats_file = self._decompress(rstats_filename)
        self._load_history()

    def _decompress(self, filename):
        try:
            return gzip.open(filename, 'rb')
        except IOError as e:
            sys.stderr.write("File could not be decompressed\n")

    def _load_history(self):
        max_daily = 62
        max_monthly = 25

        daily = []
        monthly = []

        version, = struct.unpack("I0l", self.rstats_file.read(8))

        if version == RStatsHistory.ID_V0:
           max_monthly = 12

        for i in range(max_daily):
            self.daily.append(Bandwidth._make(struct.unpack("I2Q", self.rstats_file.read(24))))

        dailyp, = struct.unpack("i0l", self.rstats_file.read(8))

        for i in range(max_monthly):
            self.monthly.append(Bandwidth._make(struct.unpack("I2Q", self.rstats_file.read(24))))

        monthlyp, = struct.unpack("i0l", self.rstats_file.read(8))

    def _get_date(self, xtime):
        return Date(
            ((xtime >> 16) & 0xFF) + 1900,
            ((xtime >>  8) & 0xFF) + 1,
            xtime & 0xFF)

    def _to_unit(self, value, unit=None):
        if unit is not None:
            unit = unit.lower()
        if unit == 'k':
            return value / 1024
        if unit == 'm':
            return value / (1024 * 1024)
        elif unit == 'g':
            return value / (1024 * 1024 * 1024)
        return value

    def _print_counters(self, counters, unit=None):
        for counter in counters:
            if counter.date != 0:
                date = self._get_date(counter.date)
                print("{0}/{1}/{2} : Upload {3} : Download {4}".format(
                    date.year, date.month, date.day,
                    self._to_unit(counter.up, unit),
                    self._to_unit(counter.down, unit)))

    def print_daily(self, unit=None):
        self._print_counters(self.daily, unit)

    def print_monthly(self, unit=None):
        self._print_counters(self.monthly, unit)

def main():
    import optparse
    from os.path import isfile

    usage = "usage: %prog [options] <filename>"
    parser = optparse.OptionParser(usage)

    parser.add_option("-u", "--unit",
                      default = None,
                      help = "Units b (bytes),k (KiB),m (MiB),g (GiB) default is b")

    options, args = parser.parse_args()

    unit = None
    if options.unit:
        unit = options.unit

    if len(args) == 1 and isfile(args[0]):
        rstats = RStatsHistory(args[0])

        print("Daily:")
        rstats.print_daily(unit)
        print("Monthly:")
        rstats.print_monthly(unit)
    else:
        print("Incorrect arguments")
        sys.exit(2)

if __name__ == "__main__":
    main()
<commit_msg>Removed unnecessary unit variable.<commit_after>#!/usr/bin/python

from collections import namedtuple
import gzip
import struct
import sys

Bandwidth = namedtuple('Bandwidth', 'date up down')
Date = namedtuple('Date', 'year month day')

class RStatsHistory(object):
    ID_V0 = 0x30305352
    ID_V1 = 0x31305352

    def __init__(self, rstats_filename):
        self.daily = []
        self.monthly = []
        self.rstats_file = self._decompress(rstats_filename)
        self._load_history()

    def _decompress(self, filename):
        try:
            return gzip.open(filename, 'rb')
        except IOError as e:
            sys.stderr.write("File could not be decompressed\n")

    def _load_history(self):
        max_daily = 62
        max_monthly = 25

        daily = []
        monthly = []

        version, = struct.unpack("I0l", self.rstats_file.read(8))

        if version == RStatsHistory.ID_V0:
           max_monthly = 12

        for i in range(max_daily):
            self.daily.append(Bandwidth._make(struct.unpack("I2Q", self.rstats_file.read(24))))

        dailyp, = struct.unpack("i0l", self.rstats_file.read(8))

        for i in range(max_monthly):
            self.monthly.append(Bandwidth._make(struct.unpack("I2Q", self.rstats_file.read(24))))

        monthlyp, = struct.unpack("i0l", self.rstats_file.read(8))

    def _get_date(self, xtime):
        return Date(
            ((xtime >> 16) & 0xFF) + 1900,
            ((xtime >>  8) & 0xFF) + 1,
            xtime & 0xFF)

    def _to_unit(self, value, unit=None):
        if unit is not None:
            unit = unit.lower()
        if unit == 'k':
            return value / 1024
        if unit == 'm':
            return value / (1024 * 1024)
        elif unit == 'g':
            return value / (1024 * 1024 * 1024)
        return value

    def _print_counters(self, counters, unit=None):
        for counter in counters:
            if counter.date != 0:
                date = self._get_date(counter.date)
                print("{0}/{1}/{2} : Upload {3} : Download {4}".format(
                    date.year, date.month, date.day,
                    self._to_unit(counter.up, unit),
                    self._to_unit(counter.down, unit)))

    def print_daily(self, unit=None):
        self._print_counters(self.daily, unit)

    def print_monthly(self, unit=None):
        self._print_counters(self.monthly, unit)

def main():
    import optparse
    from os.path import isfile

    usage = "usage: %prog [options] <filename>"
    parser = optparse.OptionParser(usage)

    parser.add_option("-u", "--unit",
                      default = None,
                      help = "Units b (bytes),k (KiB),m (MiB),g (GiB) default is b")

    options, args = parser.parse_args()

    if len(args) == 1 and isfile(args[0]):
        rstats = RStatsHistory(args[0])

        print("Daily:")
        rstats.print_daily(options.unit)
        print("Monthly:")
        rstats.print_monthly(options.unit)
    else:
        print("Incorrect arguments")
        sys.exit(2)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before><commit_msg>learned basics. still need to work on db lazy app<commit_after><|endoftext|>


<commit_before><commit_msg>fix: Validation message<commit_after><|endoftext|>


<commit_before>## Simple electricity market examples
#
#This Jupyter notebook is meant for teaching purposes. To use it, you need to install a Python environment with Jupyter notebooks, and the Python for Power System Analysis (PyPSA) library. See
#
#https://pypsa.org/doc/installation.html
#
#for tips on installation.
#
#It gradually builds up more and more complicated energy-only electricity markets in PyPSA, starting from a single bidding zone, going up to multiple bidding zones connected with transmission (NTCs) along with variable renewables and storage.
#
#Available as a Jupyter notebook at http://www.pypsa.org/examples/simple-electricity-market-examples.ipynb.

### Preliminaries
#
#Here libraries are imported and data is defined.

import pypsa, numpy as np

#marginal costs in EUR/MWh
marginal_costs = {"Wind" : 0,
                  "Hydro" : 0,
                  "Coal" : 30,
                  "Gas" : 60,
                  "Oil" : 80}

#power plant capacities (nominal powers in MW) in each country (not necessarily realistic)
power_plant_p_nom = {"South Africa" : {"Coal" : 35000,
                                       "Wind" : 3000,
                                       "Gas" : 8000,
                                       "Oil" : 2000
                                      },
                     "Mozambique" : {"Hydro" : 1200,
                                    },
                     "Swaziland" : {"Hydro" : 600,
                                    },
                    }

#transmission capacities in MW (not necessarily realistic)
transmission = {"South Africa" : {"Mozambique" : 500,
                                  "Swaziland" : 250},
                "Mozambique" : {"Swaziland" : 100}}

#country electrical loads in MW (not necessarily realistic)
loads = {"South Africa" : 42000,
         "Mozambique" : 650,
         "Swaziland" : 250}

### Single bidding zone with fixed load, one period
#
#In this example we consider a single market bidding zone, South Africa.
#
#The inelastic load has essentially infinite marginal utility (or higher than the marginal cost of any generator).

country = "South Africa"

network = pypsa.Network()

network.add("Bus",country)

for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech])

network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country])

#Run optimisation to determine market dispatch
network.lopf()

#print the load active power (P) consumption
network.loads_t.p

#print the generator active power (P) dispatch
network.generators_t.p

### Two bidding zones connected by transmission, one period
#
#In this example we have bidirectional transmission capacity between two bidding zones. The power transfer is treated as controllable (like an A/NTC (Available/Net Transfer Capacity) or HVDC line). Note that in the physical grid, power flows passively according to the network impedances.

network = pypsa.Network()

countries = ["Mozambique", "South Africa"]

for country in countries:

    network.add("Bus",country)

    for tech in power_plant_p_nom[country]:
        network.add("Generator",
                    "{} {}".format(country,tech),
                    bus=country,
                    p_nom=power_plant_p_nom[country][tech],
                    marginal_cost=marginal_costs[tech])

    network.add("Load",
                "{} load".format(country),
                bus=country,
                p_set=loads[country])

    #add transmission as controllable Link
    if country not in transmission:
        continue

    for other_country in countries:
        if other_country not in transmission[country]:
            continue

        #NB: Link is by default unidirectional, so have to set p_min_pu = -1
        #to allow bidirectional (i.e. also negative) flow
        network.add("Link",
                   "{} - {} link".format(country, other_country),
                   bus0=country,
                   bus1=other_country,
                   p_nom=transmission[country][other_country],
                   p_min_pu=-1)

network.lopf()

network.loads_t.p

network.generators_t.p

network.links_t.p0

### Three bidding zones connected by transmission, one period
#
#In this example we have bidirectional transmission capacity between three bidding zones. The power transfer is treated as controllable (like an A/NTC (Available/Net Transfer Capacity) or HVDC line). Note that in the physical grid, power flows passively according to the network impedances.

network = pypsa.Network()

countries = ["Swaziland", "Mozambique", "South Africa"]

for country in countries:

    network.add("Bus",country)

    for tech in power_plant_p_nom[country]:
        network.add("Generator",
                    "{} {}".format(country,tech),
                    bus=country,
                    p_nom=power_plant_p_nom[country][tech],
                    marginal_cost=marginal_costs[tech])

    network.add("Load",
                "{} load".format(country),
                bus=country,
                p_set=loads[country])

    #add transmission as controllable Link
    if country not in transmission:
        continue

    for other_country in countries:
        if other_country not in transmission[country]:
            continue

        #NB: Link is by default unidirectional, so have to set p_min_pu = -1
        #to allow bidirectional (i.e. also negative) flow
        network.add("Link",
                   "{} - {} link".format(country, other_country),
                   bus0=country,
                   bus1=other_country,
                   p_nom=transmission[country][other_country],
                   p_min_pu=-1)

network.lopf()

network.loads_t.p

network.generators_t.p

network.links_t.p0

### Single bidding zone with price-sensitive industrial load, one period
#
#In this example we consider a single market bidding zone, South Africa.
#
#Now there is a large industrial load with a marginal utility which is low enough to interact with the generation marginal cost.

country = "South Africa"

network = pypsa.Network()

network.add("Bus",country)

for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech])

#standard high marginal utility consumers
network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country])

#add an industrial load as a dummy negative-dispatch generator with marginal utility of 70 EUR/MWh for 8000 MW
network.add("Generator",
            "{} industrial load".format(country),
            bus=country,
            p_max_pu=0,
            p_min_pu=-1,
            p_nom=8000,
            marginal_cost=70)

network.lopf()

network.loads_t.p

#NB only half of industrial load is served, because this maxes out
#Gas. Oil is too expensive with a marginal cost of 80 EUR/MWh
network.generators_t.p

### Single bidding zone with fixed load, several periods
#
#In this example we consider a single market bidding zone, South Africa.
#
#We consider multiple time periods (labelled [0,1,2,3]) to represent variable wind generation.

country = "South Africa"

network = pypsa.Network()

#snapshots labelled by [0,1,2,3]
network.set_snapshots(range(4))

network.add("Bus",country)

#p_max_pu is variable for wind
for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech],
                p_max_pu=([0.3,0.6,0.4,0.5] if tech == "Wind" else 1),
                )

#load which varies over the snapshots
network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country] + np.array([0,1000,3000,4000]))

#specify that we consider all snapshots
network.lopf(network.snapshots)

network.loads_t.p

network.generators_t.p

### Single bidding zone with fixed load and storage, several periods
#
#In this example we consider a single market bidding zone, South Africa.
#
#We consider multiple time periods (labelled [0,1,2,3]) to represent variable wind generation. Storage is allowed to do price arbitrage to reduce oil consumption.

country = "South Africa"

network = pypsa.Network()

#snapshots labelled by [0,1,2,3]
network.set_snapshots(range(4))

network.add("Bus",country)

#p_max_pu is variable for wind
for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech],
                p_max_pu=([0.3,0.6,0.4,0.5] if tech == "Wind" else 1),
                )

#load which varies over the snapshots
network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country] + np.array([0,1000,3000,4000]))

#storage unit to do price arbitrage
network.add("StorageUnit",
            "{} pumped hydro".format(country),
            bus=country,
            p_nom=1000,
            max_hours=6, #energy storage in terms of hours at full power
           )

network.lopf(network.snapshots)

network.loads_t.p

network.generators_t.p

network.storage_units_t.p

network.storage_units_t.state_of_charge

<commit_msg>examples: Update simple market example to print clearing price<commit_after>## Simple electricity market examples
#
#This Jupyter notebook is meant for teaching purposes. To use it, you need to install a Python environment with Jupyter notebooks, and the Python for Power System Analysis (PyPSA) library. See
#
#https://pypsa.org/doc/installation.html
#
#for tips on installation.
#
#It gradually builds up more and more complicated energy-only electricity markets in PyPSA, starting from a single bidding zone, going up to multiple bidding zones connected with transmission (NTCs) along with variable renewables and storage.
#
#Available as a Jupyter notebook at http://www.pypsa.org/examples/simple-electricity-market-examples.ipynb.

### Preliminaries
#
#Here libraries are imported and data is defined.

import pypsa, numpy as np

#marginal costs in EUR/MWh
marginal_costs = {"Wind" : 0,
                  "Hydro" : 0,
                  "Coal" : 30,
                  "Gas" : 60,
                  "Oil" : 80}

#power plant capacities (nominal powers in MW) in each country (not necessarily realistic)
power_plant_p_nom = {"South Africa" : {"Coal" : 35000,
                                       "Wind" : 3000,
                                       "Gas" : 8000,
                                       "Oil" : 2000
                                      },
                     "Mozambique" : {"Hydro" : 1200,
                                    },
                     "Swaziland" : {"Hydro" : 600,
                                    },
                    }

#transmission capacities in MW (not necessarily realistic)
transmission = {"South Africa" : {"Mozambique" : 500,
                                  "Swaziland" : 250},
                "Mozambique" : {"Swaziland" : 100}}

#country electrical loads in MW (not necessarily realistic)
loads = {"South Africa" : 42000,
         "Mozambique" : 650,
         "Swaziland" : 250}

### Single bidding zone with fixed load, one period
#
#In this example we consider a single market bidding zone, South Africa.
#
#The inelastic load has essentially infinite marginal utility (or higher than the marginal cost of any generator).

country = "South Africa"

network = pypsa.Network()

network.add("Bus",country)

for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech])

network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country])

#Run optimisation to determine market dispatch
network.lopf()

#print the load active power (P) consumption
network.loads_t.p

#print the generator active power (P) dispatch
network.generators_t.p

#print the clearing price (corresponding to gas)
network.buses_t.marginal_price

### Two bidding zones connected by transmission, one period
#
#In this example we have bidirectional transmission capacity between two bidding zones. The power transfer is treated as controllable (like an A/NTC (Available/Net Transfer Capacity) or HVDC line). Note that in the physical grid, power flows passively according to the network impedances.

network = pypsa.Network()

countries = ["Mozambique", "South Africa"]

for country in countries:

    network.add("Bus",country)

    for tech in power_plant_p_nom[country]:
        network.add("Generator",
                    "{} {}".format(country,tech),
                    bus=country,
                    p_nom=power_plant_p_nom[country][tech],
                    marginal_cost=marginal_costs[tech])

    network.add("Load",
                "{} load".format(country),
                bus=country,
                p_set=loads[country])

    #add transmission as controllable Link
    if country not in transmission:
        continue

    for other_country in countries:
        if other_country not in transmission[country]:
            continue

        #NB: Link is by default unidirectional, so have to set p_min_pu = -1
        #to allow bidirectional (i.e. also negative) flow
        network.add("Link",
                   "{} - {} link".format(country, other_country),
                   bus0=country,
                   bus1=other_country,
                   p_nom=transmission[country][other_country],
                   p_min_pu=-1)

network.lopf()

network.loads_t.p

network.generators_t.p

network.links_t.p0

#print the clearing price (corresponding to water in Mozambique and gas in SA)
network.buses_t.marginal_price

### Three bidding zones connected by transmission, one period
#
#In this example we have bidirectional transmission capacity between three bidding zones. The power transfer is treated as controllable (like an A/NTC (Available/Net Transfer Capacity) or HVDC line). Note that in the physical grid, power flows passively according to the network impedances.

network = pypsa.Network()

countries = ["Swaziland", "Mozambique", "South Africa"]

for country in countries:

    network.add("Bus",country)

    for tech in power_plant_p_nom[country]:
        network.add("Generator",
                    "{} {}".format(country,tech),
                    bus=country,
                    p_nom=power_plant_p_nom[country][tech],
                    marginal_cost=marginal_costs[tech])

    network.add("Load",
                "{} load".format(country),
                bus=country,
                p_set=loads[country])

    #add transmission as controllable Link
    if country not in transmission:
        continue

    for other_country in countries:
        if other_country not in transmission[country]:
            continue

        #NB: Link is by default unidirectional, so have to set p_min_pu = -1
        #to allow bidirectional (i.e. also negative) flow
        network.add("Link",
                   "{} - {} link".format(country, other_country),
                   bus0=country,
                   bus1=other_country,
                   p_nom=transmission[country][other_country],
                   p_min_pu=-1)

network.lopf()

network.loads_t.p

network.generators_t.p

network.links_t.p0

#print the clearing price (corresponding to hydro in S and M, and gas in SA)
network.buses_t.marginal_price

### Single bidding zone with price-sensitive industrial load, one period
#
#In this example we consider a single market bidding zone, South Africa.
#
#Now there is a large industrial load with a marginal utility which is low enough to interact with the generation marginal cost.

country = "South Africa"

network = pypsa.Network()

network.add("Bus",country)

for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech])

#standard high marginal utility consumers
network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country])

#add an industrial load as a dummy negative-dispatch generator with marginal utility of 70 EUR/MWh for 8000 MW
network.add("Generator",
            "{} industrial load".format(country),
            bus=country,
            p_max_pu=0,
            p_min_pu=-1,
            p_nom=8000,
            marginal_cost=70)

network.lopf()

network.loads_t.p

#NB only half of industrial load is served, because this maxes out
#Gas. Oil is too expensive with a marginal cost of 80 EUR/MWh
network.generators_t.p

network.buses_t.marginal_price

### Single bidding zone with fixed load, several periods
#
#In this example we consider a single market bidding zone, South Africa.
#
#We consider multiple time periods (labelled [0,1,2,3]) to represent variable wind generation.

country = "South Africa"

network = pypsa.Network()

#snapshots labelled by [0,1,2,3]
network.set_snapshots(range(4))

network.add("Bus",country)

#p_max_pu is variable for wind
for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech],
                p_max_pu=([0.3,0.6,0.4,0.5] if tech == "Wind" else 1),
                )

#load which varies over the snapshots
network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country] + np.array([0,1000,3000,4000]))

#specify that we consider all snapshots
network.lopf(network.snapshots)

network.loads_t.p

network.generators_t.p

network.buses_t.marginal_price

### Single bidding zone with fixed load and storage, several periods
#
#In this example we consider a single market bidding zone, South Africa.
#
#We consider multiple time periods (labelled [0,1,2,3]) to represent variable wind generation. Storage is allowed to do price arbitrage to reduce oil consumption.

country = "South Africa"

network = pypsa.Network()

#snapshots labelled by [0,1,2,3]
network.set_snapshots(range(4))

network.add("Bus",country)

#p_max_pu is variable for wind
for tech in power_plant_p_nom[country]:
    network.add("Generator",
                "{} {}".format(country,tech),
                bus=country,
                p_nom=power_plant_p_nom[country][tech],
                marginal_cost=marginal_costs[tech],
                p_max_pu=([0.3,0.6,0.4,0.5] if tech == "Wind" else 1),
                )

#load which varies over the snapshots
network.add("Load",
            "{} load".format(country),
            bus=country,
            p_set=loads[country] + np.array([0,1000,3000,4000]))

#storage unit to do price arbitrage
network.add("StorageUnit",
            "{} pumped hydro".format(country),
            bus=country,
            p_nom=1000,
            max_hours=6, #energy storage in terms of hours at full power
           )

network.lopf(network.snapshots)

network.loads_t.p

network.generators_t.p

network.storage_units_t.p

network.storage_units_t.state_of_charge

network.buses_t.marginal_price

<|endoftext|>


<commit_before># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from typing import Dict, List, Optional, Tuple

import torch
import torch.nn as nn
from torch import Tensor

from fairseq import utils
from fairseq.dataclass.utils import gen_parser_from_dataclass
from fairseq.distributed import fsdp_wrap
from fairseq.models import FairseqEncoderDecoderModel
from fairseq.models.transformer import (
    TransformerConfig,
    TransformerDecoderBase,
    TransformerEncoderBase,
)

class TransformerModelBase(FairseqEncoderDecoderModel):
    """
    Transformer model from `"Attention Is All You Need" (Vaswani, et al, 2017)
    <https://arxiv.org/abs/1706.03762>`_.

    Args:
        encoder (TransformerEncoder): the encoder
        decoder (TransformerDecoder): the decoder

    The Transformer model provides the following named architectures and
    command-line arguments:

    .. argparse::
        :ref: fairseq.models.transformer_parser
        :prog:
    """

    def __init__(self, cfg, encoder, decoder):
        super().__init__(encoder, decoder)
        self.cfg = cfg
        self.supports_align_args = True

    @classmethod
    def add_args(cls, parser):
        """Add model-specific arguments to the parser."""
        # we want to build the args recursively in this case.
        gen_parser_from_dataclass(
            parser, TransformerConfig(), delete_default=False, with_prefix=""
        )

    @classmethod
    def build_model(cls, cfg, task):
        """Build a new model instance."""

        # --  TODO T96535332
        #  bug caused by interaction between OmegaConf II and argparsing
        cfg.decoder.input_dim = int(cfg.decoder.input_dim)
        cfg.decoder.output_dim = int(cfg.decoder.output_dim)
        # --

        if cfg.encoder.layers_to_keep:
            cfg.encoder.layers = len(cfg.encoder.layers_to_keep.split(","))
        if cfg.decoder.layers_to_keep:
            cfg.decoder.layers = len(cfg.decoder.layers_to_keep.split(","))

        src_dict, tgt_dict = task.source_dictionary, task.target_dictionary

        if cfg.share_all_embeddings:
            if src_dict != tgt_dict:
                raise ValueError("--share-all-embeddings requires a joined dictionary")
            if cfg.encoder.embed_dim != cfg.decoder.embed_dim:
                raise ValueError(
                    "--share-all-embeddings requires --encoder-embed-dim to match --decoder-embed-dim"
                )
            if cfg.decoder.embed_path and (
                cfg.decoder.embed_path != cfg.encoder.embed_path
            ):
                raise ValueError(
                    "--share-all-embeddings not compatible with --decoder-embed-path"
                )
            encoder_embed_tokens = cls.build_embedding(
                cfg, src_dict, cfg.encoder.embed_dim, cfg.encoder.embed_path
            )
            decoder_embed_tokens = encoder_embed_tokens
            cfg.share_decoder_input_output_embed = True
        else:
            encoder_embed_tokens = cls.build_embedding(
                cfg, src_dict, cfg.encoder.embed_dim, cfg.encoder.embed_path
            )
            decoder_embed_tokens = cls.build_embedding(
                cfg, tgt_dict, cfg.decoder.embed_dim, cfg.decoder.embed_path
            )
        if cfg.offload_activations:
            cfg.checkpoint_activations = True  # offloading implies checkpointing
        encoder = cls.build_encoder(cfg, src_dict, encoder_embed_tokens)
        decoder = cls.build_decoder(cfg, tgt_dict, decoder_embed_tokens)
        if not cfg.share_all_embeddings:
            # fsdp_wrap is a no-op when --ddp-backend != fully_sharded
            encoder = fsdp_wrap(encoder, min_num_params=cfg.min_params_to_wrap)
            decoder = fsdp_wrap(decoder, min_num_params=cfg.min_params_to_wrap)
        return cls(cfg, encoder, decoder)

    @classmethod
    def build_embedding(cls, cfg, dictionary, embed_dim, path=None):
        num_embeddings = len(dictionary)
        padding_idx = dictionary.pad()

        emb = Embedding(num_embeddings, embed_dim, padding_idx)
        # if provided, load from preloaded dictionaries
        if path:
            embed_dict = utils.parse_embedding(path)
            utils.load_embedding(embed_dict, dictionary, emb)
        return emb

    @classmethod
    def build_encoder(cls, cfg, src_dict, embed_tokens):
        return TransformerEncoderBase(cfg, src_dict, embed_tokens)

    @classmethod
    def build_decoder(cls, cfg, tgt_dict, embed_tokens):
        return TransformerDecoderBase(
            cfg,
            tgt_dict,
            embed_tokens,
            no_encoder_attn=cfg.no_cross_attention,
        )

    # TorchScript doesn't support optional arguments with variable length (**kwargs).
    # Current workaround is to add union of all arguments in child classes.
    def forward(
        self,
        src_tokens,
        src_lengths,
        prev_output_tokens,
        return_all_hiddens: bool = True,
        features_only: bool = False,
        alignment_layer: Optional[int] = None,
        alignment_heads: Optional[int] = None,
    ):
        """
        Run the forward pass for an encoder-decoder model.

        Copied from the base class, but without ``**kwargs``,
        which are not supported by TorchScript.
        """
        encoder_out = self.encoder(
            src_tokens, src_lengths=src_lengths, return_all_hiddens=return_all_hiddens
        )
        decoder_out = self.decoder(
            prev_output_tokens,
            encoder_out=encoder_out,
            features_only=features_only,
            alignment_layer=alignment_layer,
            alignment_heads=alignment_heads,
            src_lengths=src_lengths,
            return_all_hiddens=return_all_hiddens,
        )
        return decoder_out

    # Since get_normalized_probs is in the Fairseq Model which is not scriptable,
    # I rewrite the get_normalized_probs from Base Class to call the
    # helper function in the Base Class.
    @torch.jit.export
    def get_normalized_probs(
        self,
        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
        log_probs: bool,
        sample: Optional[Dict[str, Tensor]] = None,
    ):
        """Get normalized probabilities (or log probs) from a net's output."""
        return self.get_normalized_probs_scriptable(net_output, log_probs, sample)

def Embedding(num_embeddings, embedding_dim, padding_idx):
    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
    nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m
<commit_msg>Don't fsdp_wrap transformer encoder & decoder<commit_after># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from typing import Dict, List, Optional, Tuple

import torch
import torch.nn as nn
from torch import Tensor

from fairseq import utils
from fairseq.dataclass.utils import gen_parser_from_dataclass
from fairseq.distributed import fsdp_wrap
from fairseq.models import FairseqEncoderDecoderModel
from fairseq.models.transformer import (
    TransformerConfig,
    TransformerDecoderBase,
    TransformerEncoderBase,
)

class TransformerModelBase(FairseqEncoderDecoderModel):
    """
    Transformer model from `"Attention Is All You Need" (Vaswani, et al, 2017)
    <https://arxiv.org/abs/1706.03762>`_.

    Args:
        encoder (TransformerEncoder): the encoder
        decoder (TransformerDecoder): the decoder

    The Transformer model provides the following named architectures and
    command-line arguments:

    .. argparse::
        :ref: fairseq.models.transformer_parser
        :prog:
    """

    def __init__(self, cfg, encoder, decoder):
        super().__init__(encoder, decoder)
        self.cfg = cfg
        self.supports_align_args = True

    @classmethod
    def add_args(cls, parser):
        """Add model-specific arguments to the parser."""
        # we want to build the args recursively in this case.
        gen_parser_from_dataclass(
            parser, TransformerConfig(), delete_default=False, with_prefix=""
        )

    @classmethod
    def build_model(cls, cfg, task):
        """Build a new model instance."""

        # --  TODO T96535332
        #  bug caused by interaction between OmegaConf II and argparsing
        cfg.decoder.input_dim = int(cfg.decoder.input_dim)
        cfg.decoder.output_dim = int(cfg.decoder.output_dim)
        # --

        if cfg.encoder.layers_to_keep:
            cfg.encoder.layers = len(cfg.encoder.layers_to_keep.split(","))
        if cfg.decoder.layers_to_keep:
            cfg.decoder.layers = len(cfg.decoder.layers_to_keep.split(","))

        src_dict, tgt_dict = task.source_dictionary, task.target_dictionary

        if cfg.share_all_embeddings:
            if src_dict != tgt_dict:
                raise ValueError("--share-all-embeddings requires a joined dictionary")
            if cfg.encoder.embed_dim != cfg.decoder.embed_dim:
                raise ValueError(
                    "--share-all-embeddings requires --encoder-embed-dim to match --decoder-embed-dim"
                )
            if cfg.decoder.embed_path and (
                cfg.decoder.embed_path != cfg.encoder.embed_path
            ):
                raise ValueError(
                    "--share-all-embeddings not compatible with --decoder-embed-path"
                )
            encoder_embed_tokens = cls.build_embedding(
                cfg, src_dict, cfg.encoder.embed_dim, cfg.encoder.embed_path
            )
            decoder_embed_tokens = encoder_embed_tokens
            cfg.share_decoder_input_output_embed = True
        else:
            encoder_embed_tokens = cls.build_embedding(
                cfg, src_dict, cfg.encoder.embed_dim, cfg.encoder.embed_path
            )
            decoder_embed_tokens = cls.build_embedding(
                cfg, tgt_dict, cfg.decoder.embed_dim, cfg.decoder.embed_path
            )
        if cfg.offload_activations:
            cfg.checkpoint_activations = True  # offloading implies checkpointing
        encoder = cls.build_encoder(cfg, src_dict, encoder_embed_tokens)
        decoder = cls.build_decoder(cfg, tgt_dict, decoder_embed_tokens)
        return cls(cfg, encoder, decoder)

    @classmethod
    def build_embedding(cls, cfg, dictionary, embed_dim, path=None):
        num_embeddings = len(dictionary)
        padding_idx = dictionary.pad()

        emb = Embedding(num_embeddings, embed_dim, padding_idx)
        # if provided, load from preloaded dictionaries
        if path:
            embed_dict = utils.parse_embedding(path)
            utils.load_embedding(embed_dict, dictionary, emb)
        return emb

    @classmethod
    def build_encoder(cls, cfg, src_dict, embed_tokens):
        return TransformerEncoderBase(cfg, src_dict, embed_tokens)

    @classmethod
    def build_decoder(cls, cfg, tgt_dict, embed_tokens):
        return TransformerDecoderBase(
            cfg,
            tgt_dict,
            embed_tokens,
            no_encoder_attn=cfg.no_cross_attention,
        )

    # TorchScript doesn't support optional arguments with variable length (**kwargs).
    # Current workaround is to add union of all arguments in child classes.
    def forward(
        self,
        src_tokens,
        src_lengths,
        prev_output_tokens,
        return_all_hiddens: bool = True,
        features_only: bool = False,
        alignment_layer: Optional[int] = None,
        alignment_heads: Optional[int] = None,
    ):
        """
        Run the forward pass for an encoder-decoder model.

        Copied from the base class, but without ``**kwargs``,
        which are not supported by TorchScript.
        """
        encoder_out = self.encoder(
            src_tokens, src_lengths=src_lengths, return_all_hiddens=return_all_hiddens
        )
        decoder_out = self.decoder(
            prev_output_tokens,
            encoder_out=encoder_out,
            features_only=features_only,
            alignment_layer=alignment_layer,
            alignment_heads=alignment_heads,
            src_lengths=src_lengths,
            return_all_hiddens=return_all_hiddens,
        )
        return decoder_out

    # Since get_normalized_probs is in the Fairseq Model which is not scriptable,
    # I rewrite the get_normalized_probs from Base Class to call the
    # helper function in the Base Class.
    @torch.jit.export
    def get_normalized_probs(
        self,
        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
        log_probs: bool,
        sample: Optional[Dict[str, Tensor]] = None,
    ):
        """Get normalized probabilities (or log probs) from a net's output."""
        return self.get_normalized_probs_scriptable(net_output, log_probs, sample)

def Embedding(num_embeddings, embedding_dim, padding_idx):
    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
    nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m
<|endoftext|>