<commit_before>import os
import os.path
import re
import datetime
import unicodedata
from formatter import PageFormatter, FormattingContext
from resolver import PageResolver, CircularIncludeError

class PageData(object):
    def __init__(self):
        self.path = None
        self.title = None
        self.raw_text = None
        self.formatted_text = None
        self.text = None
        self.local_meta = {}
        self.local_links = []
        self.ext_meta = {}
        self.ext_links = []
        self.has_extended_data = False

class Page(object):
    """ A wiki page.
    """
    def __init__(self, wiki, url):
        self.wiki = wiki
        self.url = url
        self._data = None

    @property
    def path(self):
        self._ensureData()
        return self._data.path

    @property
    def extension(self):
        self._ensureData()
        return self._data.extension

    @property
    def filename(self):
        self._ensureData()
        return self._data.filename

    @property
    def title(self):
        self._ensureData()
        return self._data.title

    @property
    def raw_text(self):
        self._ensureData()
        return self._data.raw_text

    @property
    def text(self):
        self._ensureExtendedData()
        return self._data.text

    @property
    def meta(self):
        self._ensureExtendedData()
        return self._data.ext_meta

    @property
    def links(self):
        self._ensureExtendedData()
        return self._data.ext_links

    def getIncomingLinks(self):
        return self.wiki.db.getLinksTo(self.url)

    def getHistory(self):
        return self.wiki.scm.getHistory(self.path)

    def getState(self):
        return self.wiki.scm.getState(self.path)

    def getRevision(self, rev):
        return self.wiki.scm.getRevision(self.path, rev)

    def getDiff(self, rev1, rev2):
        return self.wiki.scm.diff(self.path, rev1, rev2)

    def _getFormattedText(self):
        self._ensureData()
        return self._data.formatted_text

    def _getLocalMeta(self):
        self._ensureData()
        return self._data.local_meta

    def _getLocalLinks(self):
        self._ensureData()
        return self._data.local_links

    def _ensureData(self):
        if self._data is not None:
            return

        self._data = self._loadCachedData()
        if self._data is not None:
            return

        self._data = self._loadOriginalData()
        self._saveCachedData(self._data)

    def _loadCachedData(self):
        return None

    def _saveCachedData(self, meta):
        pass

    def _loadOriginalData(self):
        data = PageData()

        # Get info from the file-system.
        page_info = self.wiki.fs.getPage(self.url)
        data.path = page_info.path
        data.raw_text = page_info.content
        split = os.path.splitext(data.path)
        data.filename = split[0]
        data.extension = split[1].lstrip('.')

        # Format the page and get the meta properties.
        filename = os.path.basename(data.path)
        filename_split = os.path.splitext(filename)
        ctx = FormattingContext(self.url, slugify=Page.title_to_url)
        f = PageFormatter(self.wiki)
        data.formatted_text = f.formatText(ctx, data.raw_text)
        data.local_meta = ctx.meta
        data.local_links = ctx.out_links

        # Add some common meta.
        data.title = re.sub(r'\-', ' ', filename_split[0])
        if 'title' in data.local_meta:
            data.title = data.local_meta['title'][0]

        return data

    def _ensureExtendedData(self):
        if self._data is not None and self._data.has_extended_data:
            return

        self._ensureData()
        try:
            r = PageResolver(self)
            out = r.run()
            self._data.text = out.text
            self._data.ext_meta = out.meta
            self._data.ext_links = out.out_links
        except CircularIncludeError as cie:
            template_path = os.path.join(
                    os.path.dirname(__file__),
                    'templates',
                    'circular_include_error.html'
                    )
            with open(template_path, 'r') as f:
                env = jinja2.Environment()
                template = env.from_string(f.read())
            self._data.text = template.render({
                    'message': str(cie),
                    'url_trail': cie.url_trail
                    })

    @staticmethod
    def title_to_url(title):
        # Remove diacritics (accents, etc.) and replace them with ASCII
        # equivelent.
        ansi_title = ''.join((c for c in
            unicodedata.normalize('NFD', unicode(title))
            if unicodedata.category(c) != 'Mn'))
        # Now replace spaces and punctuation with a hyphen.
        return re.sub(r'[^A-Za-z0-9_\.\-\(\)]+', '-', ansi_title.lower())

    @staticmethod
    def url_to_title(url):
        def upperChar(m):
            return m.group(0).upper()
        return re.sub(r'^.|\s\S', upperChar, url.lower().replace('-', ' '))

    @staticmethod
    def factory(wiki, url):
        return Page(wiki, url)

class DatabasePage(Page):
    """ A page that can load its properties from a
        database.
    """
    def __init__(self, wiki, url):
        Page.__init__(self, wiki, url)
        if getattr(wiki, 'db', None) is None:
            raise Exception("The wiki doesn't have a database.")
        self.auto_update = wiki.config.get('wiki', 'auto_update')

    def _loadCachedData(self):
        if self.wiki.db is None:
            return None
        db_page = self.wiki.db.getPage(self.url)
        if db_page is None:
            return None
        if self.auto_update:
            path_time = datetime.datetime.fromtimestamp(
                os.path.getmtime(db_page.path))
            if path_time >= db_page.time:
                return None
        data = PageData()
        data.path = db_page.path
        split = os.path.splitext(data.path)
        data.filename = split[0]
        data.extension = split[1].lstrip('.')
        data.title = db_page.title
        data.raw_text = db_page.raw_text
        data.formatted_text = db_page.formatted_text
        data.local_meta = db_page.meta
        data.local_links = db_page.links
        return data

    def _saveCachedData(self, meta):
        if self.wiki.db is not None:
            self.wiki.logger.debug(
                "Updated database cache for page '%s'." % self.url)
            self.wiki.db.update([self])

    @staticmethod
    def factory(wiki, url):
        return DatabasePage(wiki, url)
<commit_msg>Fixed missing import<commit_after>import os
import os.path
import re
import datetime
import unicodedata
import jinja2
from formatter import PageFormatter, FormattingContext
from resolver import PageResolver, CircularIncludeError

class PageData(object):
    def __init__(self):
        self.path = None
        self.title = None
        self.raw_text = None
        self.formatted_text = None
        self.text = None
        self.local_meta = {}
        self.local_links = []
        self.ext_meta = {}
        self.ext_links = []
        self.has_extended_data = False

class Page(object):
    """ A wiki page.
    """
    def __init__(self, wiki, url):
        self.wiki = wiki
        self.url = url
        self._data = None

    @property
    def path(self):
        self._ensureData()
        return self._data.path

    @property
    def extension(self):
        self._ensureData()
        return self._data.extension

    @property
    def filename(self):
        self._ensureData()
        return self._data.filename

    @property
    def title(self):
        self._ensureData()
        return self._data.title

    @property
    def raw_text(self):
        self._ensureData()
        return self._data.raw_text

    @property
    def text(self):
        self._ensureExtendedData()
        return self._data.text

    @property
    def meta(self):
        self._ensureExtendedData()
        return self._data.ext_meta

    @property
    def links(self):
        self._ensureExtendedData()
        return self._data.ext_links

    def getIncomingLinks(self):
        return self.wiki.db.getLinksTo(self.url)

    def getHistory(self):
        return self.wiki.scm.getHistory(self.path)

    def getState(self):
        return self.wiki.scm.getState(self.path)

    def getRevision(self, rev):
        return self.wiki.scm.getRevision(self.path, rev)

    def getDiff(self, rev1, rev2):
        return self.wiki.scm.diff(self.path, rev1, rev2)

    def _getFormattedText(self):
        self._ensureData()
        return self._data.formatted_text

    def _getLocalMeta(self):
        self._ensureData()
        return self._data.local_meta

    def _getLocalLinks(self):
        self._ensureData()
        return self._data.local_links

    def _ensureData(self):
        if self._data is not None:
            return

        self._data = self._loadCachedData()
        if self._data is not None:
            return

        self._data = self._loadOriginalData()
        self._saveCachedData(self._data)

    def _loadCachedData(self):
        return None

    def _saveCachedData(self, meta):
        pass

    def _loadOriginalData(self):
        data = PageData()

        # Get info from the file-system.
        page_info = self.wiki.fs.getPage(self.url)
        data.path = page_info.path
        data.raw_text = page_info.content
        split = os.path.splitext(data.path)
        data.filename = split[0]
        data.extension = split[1].lstrip('.')

        # Format the page and get the meta properties.
        filename = os.path.basename(data.path)
        filename_split = os.path.splitext(filename)
        ctx = FormattingContext(self.url, slugify=Page.title_to_url)
        f = PageFormatter(self.wiki)
        data.formatted_text = f.formatText(ctx, data.raw_text)
        data.local_meta = ctx.meta
        data.local_links = ctx.out_links

        # Add some common meta.
        data.title = re.sub(r'\-', ' ', filename_split[0])
        if 'title' in data.local_meta:
            data.title = data.local_meta['title'][0]

        return data

    def _ensureExtendedData(self):
        if self._data is not None and self._data.has_extended_data:
            return

        self._ensureData()
        try:
            r = PageResolver(self)
            out = r.run()
            self._data.text = out.text
            self._data.ext_meta = out.meta
            self._data.ext_links = out.out_links
        except CircularIncludeError as cie:
            template_path = os.path.join(
                    os.path.dirname(__file__),
                    'templates',
                    'circular_include_error.html'
                    )
            with open(template_path, 'r') as f:
                env = jinja2.Environment()
                template = env.from_string(f.read())
            self._data.text = template.render({
                    'message': str(cie),
                    'url_trail': cie.url_trail
                    })

    @staticmethod
    def title_to_url(title):
        # Remove diacritics (accents, etc.) and replace them with ASCII
        # equivelent.
        ansi_title = ''.join((c for c in
            unicodedata.normalize('NFD', unicode(title))
            if unicodedata.category(c) != 'Mn'))
        # Now replace spaces and punctuation with a hyphen.
        return re.sub(r'[^A-Za-z0-9_\.\-\(\)]+', '-', ansi_title.lower())

    @staticmethod
    def url_to_title(url):
        def upperChar(m):
            return m.group(0).upper()
        return re.sub(r'^.|\s\S', upperChar, url.lower().replace('-', ' '))

    @staticmethod
    def factory(wiki, url):
        return Page(wiki, url)

class DatabasePage(Page):
    """ A page that can load its properties from a
        database.
    """
    def __init__(self, wiki, url):
        Page.__init__(self, wiki, url)
        if getattr(wiki, 'db', None) is None:
            raise Exception("The wiki doesn't have a database.")
        self.auto_update = wiki.config.get('wiki', 'auto_update')

    def _loadCachedData(self):
        if self.wiki.db is None:
            return None
        db_page = self.wiki.db.getPage(self.url)
        if db_page is None:
            return None
        if self.auto_update:
            path_time = datetime.datetime.fromtimestamp(
                os.path.getmtime(db_page.path))
            if path_time >= db_page.time:
                return None
        data = PageData()
        data.path = db_page.path
        split = os.path.splitext(data.path)
        data.filename = split[0]
        data.extension = split[1].lstrip('.')
        data.title = db_page.title
        data.raw_text = db_page.raw_text
        data.formatted_text = db_page.formatted_text
        data.local_meta = db_page.meta
        data.local_links = db_page.links
        return data

    def _saveCachedData(self, meta):
        if self.wiki.db is not None:
            self.wiki.logger.debug(
                "Updated database cache for page '%s'." % self.url)
            self.wiki.db.update([self])

    @staticmethod
    def factory(wiki, url):
        return DatabasePage(wiki, url)
<|endoftext|>


<commit_before># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

class Platform:
    ''' Enumeration of supported platforms '''
    LINUX = 'linux'
    WINDOWS = 'windows'
    DARWIN = 'darwin'
    ANDROID = 'android'
    IOS = 'ios'

class Architecture:
    ''' Enumeration of supported acrchitectures '''
    X86 = 'x86'
    X86_64 = 'x86_64'
    UNIVERSAL = 'universal'
    ARM = 'arm'
    ARMv7 = 'armv7'
    ARMv7S = 'armv7s'
    ARM64 = 'arm64'

    @staticmethod
    def is_arm(arch):
        '''Returns whether the architecture is an ARM based one.
        Note that it will include 32bit *and* 64bit ARM targets. If you
        wish to do something special for 64bit you should first check for
        that before calling this method.'''
        return arch in [Architecture.ARM, Architecture.ARMv7,
                Architecture.ARMv7S, Architecture.ARM64]

    @staticmethod
    def is_arm32(arch):
        return arch in [Architecture.ARM, Architecture.ARMv7, Architecture.ARMv7S]

class Distro:
    ''' Enumeration of supported distributions '''
    DEBIAN = 'debian'
    REDHAT = 'redhat'
    SUSE = 'suse'
    WINDOWS = 'windows'
    ARCH = 'arch'
    OS_X = 'osx'
    IOS = 'ios'
    ANDROID = 'android'
    GENTOO = 'gentoo'
    NONE = 'none'

class DistroVersion:
    ''' Enumeration of supported distribution versions, withing each distro, they must be sortable'''
    DEBIAN_SQUEEZE = 'debian_06_squeeze'
    DEBIAN_WHEEZY = 'debian_07_wheezy'
    DEBIAN_JESSIE = 'debian_08_jessie'
    DEBIAN_STRETCH = 'debian_09_stretch'
    DEBIAN_BUSTER = 'debian_10_buster'
    DEBIAN_BULLSEYE = 'debian_11_bullseye'
    UBUNTU_LUCID = 'ubuntu_10_04_lucid'
    UBUNTU_MAVERICK = 'ubuntu_10_10_maverick'
    UBUNTU_NATTY = 'ubuntu_11_04_natty'
    UBUNTU_ONEIRIC = 'ubuntu_11_10_oneiric'
    UBUNTU_PRECISE = 'ubuntu_12_04_precise'
    UBUNTU_QUANTAL = 'ubuntu_12_10_quantal'
    UBUNTU_RARING = 'ubuntu_13_04_raring'
    UBUNTU_SAUCY = 'ubuntu_13_10_saucy'
    UBUNTU_TRUSTY = 'ubuntu_14_04_trusty'
    UBUNTU_UTOPIC = 'ubuntu_14_10_utopic'
    UBUNTU_VIVID = 'ubuntu_15_04_vivid'
    UBUNTU_WILY = 'ubuntu_15_10_wily'
    UBUNTU_XENIAL = 'ubuntu_16_04_xenial'
    UBUNTU_ARTFUL = 'ubuntu_17_10_artful'
    UBUNTU_BIONIC = 'ubuntu_18_04_bionic'
    UBUNTU_COSMIC = 'ubuntu_18_10_cosmic'
    UBUNTU_DISCO = 'ubuntu_19_04_disco'
    UBUNTU_EOAN = 'ubuntu_19_10_eoan'
    FEDORA_16 = 'fedora_16'
    FEDORA_17 = 'fedora_17'
    FEDORA_18 = 'fedora_18'
    FEDORA_19 = 'fedora_19'
    FEDORA_20 = 'fedora_20'
    FEDORA_21 = 'fedora_21'
    FEDORA_22 = 'fedora_22'
    FEDORA_23 = 'fedora_23'
    FEDORA_24 = 'fedora_24'
    FEDORA_25 = 'fedora_25'
    FEDORA_26 = 'fedora_26'
    FEDORA_27 = 'fedora_27'
    FEDORA_28 = 'fedora_28'
    FEDORA_29 = 'fedora_29'
    FEDORA_30 = 'fedora_30'
    FEDORA_31 = 'fedora_31'
    REDHAT_6 = 'redhat_6'
    REDHAT_7 = 'redhat_7'
    REDHAT_8 = 'redhat_8'
    # Amazon Linux seems to be RedHat/CentOS-based
    AMAZON_LINUX = 'amazon_linux'
    ARCH_ROLLING = 'rolling'
    GENTOO_VERSION = 'gentoo-version'
    OPENSUSE_42_2 = 'opensuse_42_2'
    OPENSUSE_42_3 = 'opensuse_42_3'
    OPENSUSE_TUMBLEWEED = 'tumbleweed'
    WINDOWS_XP = 'windows_xp'
    WINDOWS_VISTA = 'windows_vista'
    WINDOWS_7 = 'windows_07'
    WINDOWS_8 = 'windows_08'
    WINDOWS_8_1 = 'windows_08_1'
    WINDOWS_10 = 'windows_10'
    OS_X_MAVERICKS = 'osx_mavericks'
    OS_X_MOUNTAIN_LION = 'osx_mountain_lion'
    OS_X_YOSEMITE = 'osx_yosemite'
    OS_X_EL_CAPITAN = 'osx_el_capitan'
    OS_X_SIERRA = 'osx_sierra'
    OS_X_HIGH_SIERRA = 'osx_high_sierra'
    OS_X_MOJAVE = 'osx_mojave'
    OS_X_CATALINA = 'osx_catalina'
    IOS_8_0 = 'ios_08_0'
    IOS_8_1 = 'ios_08_1'
    IOS_8_2 = 'ios_08_2'
    IOS_8_3 = 'ios_08_3'
    IOS_8_4 = 'ios_08_4'
    IOS_9_0 = 'ios_09_0'
    IOS_9_1 = 'ios_09_1'
    IOS_9_2 = 'ios_09_2'
    IOS_9_3 = 'ios_09_3'
    IOS_10_0 = 'ios_10_0'
    IOS_10_1 = 'ios_10_1'
    IOS_10_2 = 'ios_10_2'
    IOS_10_3 = 'ios_10_3'
    IOS_11_0 = 'ios_11_0'
    IOS_11_1 = 'ios_11_1'
    IOS_11_2 = 'ios_11_2'
    IOS_11_3 = 'ios_11_3'
    IOS_11_4 = 'ios_11_4'
    IOS_12_0 = 'ios_12_0'
    IOS_12_1 = 'ios_12_1'
    IOS_12_2 = 'ios_12_2'
    IOS_12_3 = 'ios_12_3'
    IOS_12_4 = 'ios_12_4'
    # further ios versions are generated automatically
    ANDROID_GINGERBREAD = 'android_09_gingerbread'  # API Level 9
    ANDROID_ICE_CREAM_SANDWICH = 'android_14_ice_cream_sandwich'  # API Level 14
    ANDROID_JELLY_BEAN = 'android_16_jelly_bean'  # API Level 16
    ANDROID_KITKAT = 'android_19_kitkat'  # API Level 19
    ANDROID_LOLLIPOP = 'android_21_lollipop' # API Level 21
    ANDROID_LOLLIPOP_MR1 = 'android_22_lollipop_mr1' # API Level 22
    ANDROID_MARSHMALLOW = 'android_23_marshmallow' # API Level 23
    ANDROID_NOUGAT = 'android_24_nougat' # API Level 24
    NONE_UCLIBC = 'none_uclibc'
    NONE_GLIBC = 'none_glibc'

    @staticmethod
    def get_android_api_version(version):
        '''Returns the corresponding android api version'''
        if version == DistroVersion.ANDROID_GINGERBREAD:
            return 9
        elif version == DistroVersion.ANDROID_ICE_CREAM_SANDWICH:
            return 14
        elif version == DistroVersion.ANDROID_JELLY_BEAN:
            return 16
        elif version == DistroVersion.ANDROID_KITKAT:
            return 19
        elif version == DistroVersion.ANDROID_LOLLIPOP:
            return 21
        elif version == DistroVersion.ANDROID_LOLLIPOP_MR1:
            return 22
        elif version == DistroVersion.ANDROID_MARSHMALLOW:
            return 23
        elif version == DistroVersion.ANDROID_NOUGAT:
            return 24
        else:
            raise FatalError("DistroVersion not supported")

    @staticmethod
    def get_ios_sdk_version(version):
        if not version.startswith('ios_'):
            raise FatalError('Not an iOS version: ' + version)
        return [int(s) for s in version[4:].split('_')]

class LicenseDescription:

    def __init__(self, acronym, pretty_name):
        self.acronym = acronym
        self.pretty_name = pretty_name

    def __lt__(self, other):
        return self.acronym < other.acronym

    def __repr__(self):
        return "LicenseDescription(%s)" % self.acronym

class License:
    ''' Enumeration of licensesversions '''
    Apachev2 = LicenseDescription('Apache-2.0',
            'Apache License, version 2.0')
    BSD = LicenseDescription('BSD',
            'BSD License')
    BSD_like = LicenseDescription('BSD-like',
            'BSD-like License')
    FreeType = LicenseDescription('FreeType',
            'FreeType License')
    GPLv2Plus = LicenseDescription('GPL-2+',
            'GNU General Public License, version 2 or later')
    GPLv3Plus = LicenseDescription('GPL-3+',
            'GNU General Public License, version 3 or later')
    LGPLv2Plus = LicenseDescription('LGPL-2+',
            'GNU Lesser General Public License, version 2 or later')
    LGPLv2_1Plus = LicenseDescription('LGPL-2.1+',
            'GNU Lesser General Public License, version 2.1 or later')
    LGPLv3Plus = LicenseDescription('LGPL-3+',
            'GNU Lesser General Public License, version 3 or later')
    LibPNG = LicenseDescription('LibPNG',
            'LibPNG License')
    MPLv1_1 = LicenseDescription('MPL-1.1',
            'Mozilla Public License Version 1.1')
    MPLv2 = LicenseDescription('MPL-2',
            'Mozilla Public License Version 2.0')
    MIT = LicenseDescription('MIT',
            'MIT License')
    OPENSSL = LicenseDescription('OpenSSL',
            'OpenSSL License')
    Proprietary = LicenseDescription('Proprietary',
            'Proprietary License')
    Misc = LicenseDescription('Misc',
            'Miscellaneous license information')

class LibraryType:
    NONE = 'none'
    STATIC = 'static'
    SHARED = 'shared'
    BOTH = 'both'
<commit_msg>add support Android Nougat MR1<commit_after># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

class Platform:
    ''' Enumeration of supported platforms '''
    LINUX = 'linux'
    WINDOWS = 'windows'
    DARWIN = 'darwin'
    ANDROID = 'android'
    IOS = 'ios'

class Architecture:
    ''' Enumeration of supported acrchitectures '''
    X86 = 'x86'
    X86_64 = 'x86_64'
    UNIVERSAL = 'universal'
    ARM = 'arm'
    ARMv7 = 'armv7'
    ARMv7S = 'armv7s'
    ARM64 = 'arm64'

    @staticmethod
    def is_arm(arch):
        '''Returns whether the architecture is an ARM based one.
        Note that it will include 32bit *and* 64bit ARM targets. If you
        wish to do something special for 64bit you should first check for
        that before calling this method.'''
        return arch in [Architecture.ARM, Architecture.ARMv7,
                Architecture.ARMv7S, Architecture.ARM64]

    @staticmethod
    def is_arm32(arch):
        return arch in [Architecture.ARM, Architecture.ARMv7, Architecture.ARMv7S]

class Distro:
    ''' Enumeration of supported distributions '''
    DEBIAN = 'debian'
    REDHAT = 'redhat'
    SUSE = 'suse'
    WINDOWS = 'windows'
    ARCH = 'arch'
    OS_X = 'osx'
    IOS = 'ios'
    ANDROID = 'android'
    GENTOO = 'gentoo'
    NONE = 'none'

class DistroVersion:
    ''' Enumeration of supported distribution versions, withing each distro, they must be sortable'''
    DEBIAN_SQUEEZE = 'debian_06_squeeze'
    DEBIAN_WHEEZY = 'debian_07_wheezy'
    DEBIAN_JESSIE = 'debian_08_jessie'
    DEBIAN_STRETCH = 'debian_09_stretch'
    DEBIAN_BUSTER = 'debian_10_buster'
    DEBIAN_BULLSEYE = 'debian_11_bullseye'
    UBUNTU_LUCID = 'ubuntu_10_04_lucid'
    UBUNTU_MAVERICK = 'ubuntu_10_10_maverick'
    UBUNTU_NATTY = 'ubuntu_11_04_natty'
    UBUNTU_ONEIRIC = 'ubuntu_11_10_oneiric'
    UBUNTU_PRECISE = 'ubuntu_12_04_precise'
    UBUNTU_QUANTAL = 'ubuntu_12_10_quantal'
    UBUNTU_RARING = 'ubuntu_13_04_raring'
    UBUNTU_SAUCY = 'ubuntu_13_10_saucy'
    UBUNTU_TRUSTY = 'ubuntu_14_04_trusty'
    UBUNTU_UTOPIC = 'ubuntu_14_10_utopic'
    UBUNTU_VIVID = 'ubuntu_15_04_vivid'
    UBUNTU_WILY = 'ubuntu_15_10_wily'
    UBUNTU_XENIAL = 'ubuntu_16_04_xenial'
    UBUNTU_ARTFUL = 'ubuntu_17_10_artful'
    UBUNTU_BIONIC = 'ubuntu_18_04_bionic'
    UBUNTU_COSMIC = 'ubuntu_18_10_cosmic'
    UBUNTU_DISCO = 'ubuntu_19_04_disco'
    UBUNTU_EOAN = 'ubuntu_19_10_eoan'
    FEDORA_16 = 'fedora_16'
    FEDORA_17 = 'fedora_17'
    FEDORA_18 = 'fedora_18'
    FEDORA_19 = 'fedora_19'
    FEDORA_20 = 'fedora_20'
    FEDORA_21 = 'fedora_21'
    FEDORA_22 = 'fedora_22'
    FEDORA_23 = 'fedora_23'
    FEDORA_24 = 'fedora_24'
    FEDORA_25 = 'fedora_25'
    FEDORA_26 = 'fedora_26'
    FEDORA_27 = 'fedora_27'
    FEDORA_28 = 'fedora_28'
    FEDORA_29 = 'fedora_29'
    FEDORA_30 = 'fedora_30'
    FEDORA_31 = 'fedora_31'
    REDHAT_6 = 'redhat_6'
    REDHAT_7 = 'redhat_7'
    REDHAT_8 = 'redhat_8'
    # Amazon Linux seems to be RedHat/CentOS-based
    AMAZON_LINUX = 'amazon_linux'
    ARCH_ROLLING = 'rolling'
    GENTOO_VERSION = 'gentoo-version'
    OPENSUSE_42_2 = 'opensuse_42_2'
    OPENSUSE_42_3 = 'opensuse_42_3'
    OPENSUSE_TUMBLEWEED = 'tumbleweed'
    WINDOWS_XP = 'windows_xp'
    WINDOWS_VISTA = 'windows_vista'
    WINDOWS_7 = 'windows_07'
    WINDOWS_8 = 'windows_08'
    WINDOWS_8_1 = 'windows_08_1'
    WINDOWS_10 = 'windows_10'
    OS_X_MAVERICKS = 'osx_mavericks'
    OS_X_MOUNTAIN_LION = 'osx_mountain_lion'
    OS_X_YOSEMITE = 'osx_yosemite'
    OS_X_EL_CAPITAN = 'osx_el_capitan'
    OS_X_SIERRA = 'osx_sierra'
    OS_X_HIGH_SIERRA = 'osx_high_sierra'
    OS_X_MOJAVE = 'osx_mojave'
    OS_X_CATALINA = 'osx_catalina'
    IOS_8_0 = 'ios_08_0'
    IOS_8_1 = 'ios_08_1'
    IOS_8_2 = 'ios_08_2'
    IOS_8_3 = 'ios_08_3'
    IOS_8_4 = 'ios_08_4'
    IOS_9_0 = 'ios_09_0'
    IOS_9_1 = 'ios_09_1'
    IOS_9_2 = 'ios_09_2'
    IOS_9_3 = 'ios_09_3'
    IOS_10_0 = 'ios_10_0'
    IOS_10_1 = 'ios_10_1'
    IOS_10_2 = 'ios_10_2'
    IOS_10_3 = 'ios_10_3'
    IOS_11_0 = 'ios_11_0'
    IOS_11_1 = 'ios_11_1'
    IOS_11_2 = 'ios_11_2'
    IOS_11_3 = 'ios_11_3'
    IOS_11_4 = 'ios_11_4'
    IOS_12_0 = 'ios_12_0'
    IOS_12_1 = 'ios_12_1'
    IOS_12_2 = 'ios_12_2'
    IOS_12_3 = 'ios_12_3'
    IOS_12_4 = 'ios_12_4'
    # further ios versions are generated automatically
    ANDROID_GINGERBREAD = 'android_09_gingerbread'  # API Level 9
    ANDROID_ICE_CREAM_SANDWICH = 'android_14_ice_cream_sandwich'  # API Level 14
    ANDROID_JELLY_BEAN = 'android_16_jelly_bean'  # API Level 16
    ANDROID_KITKAT = 'android_19_kitkat'  # API Level 19
    ANDROID_LOLLIPOP = 'android_21_lollipop' # API Level 21
    ANDROID_LOLLIPOP_MR1 = 'android_22_lollipop_mr1' # API Level 22
    ANDROID_MARSHMALLOW = 'android_23_marshmallow' # API Level 23
    ANDROID_NOUGAT = 'android_24_nougat' # API Level 24
    ANDROID_NOUGAT_MR1 = 'android_25_nougat_mr1' # API Level 25
    NONE_UCLIBC = 'none_uclibc'
    NONE_GLIBC = 'none_glibc'

    @staticmethod
    def get_android_api_version(version):
        '''Returns the corresponding android api version'''
        if version == DistroVersion.ANDROID_GINGERBREAD:
            return 9
        elif version == DistroVersion.ANDROID_ICE_CREAM_SANDWICH:
            return 14
        elif version == DistroVersion.ANDROID_JELLY_BEAN:
            return 16
        elif version == DistroVersion.ANDROID_KITKAT:
            return 19
        elif version == DistroVersion.ANDROID_LOLLIPOP:
            return 21
        elif version == DistroVersion.ANDROID_LOLLIPOP_MR1:
            return 22
        elif version == DistroVersion.ANDROID_MARSHMALLOW:
            return 23
        elif version == DistroVersion.ANDROID_NOUGAT:
            return 24
        elif version == DistroVersion.ANDROID_NOUGAT_MR1:
            return 25
        else:
            raise FatalError("DistroVersion not supported")

    @staticmethod
    def get_ios_sdk_version(version):
        if not version.startswith('ios_'):
            raise FatalError('Not an iOS version: ' + version)
        return [int(s) for s in version[4:].split('_')]

class LicenseDescription:

    def __init__(self, acronym, pretty_name):
        self.acronym = acronym
        self.pretty_name = pretty_name

    def __lt__(self, other):
        return self.acronym < other.acronym

    def __repr__(self):
        return "LicenseDescription(%s)" % self.acronym

class License:
    ''' Enumeration of licensesversions '''
    Apachev2 = LicenseDescription('Apache-2.0',
            'Apache License, version 2.0')
    BSD = LicenseDescription('BSD',
            'BSD License')
    BSD_like = LicenseDescription('BSD-like',
            'BSD-like License')
    FreeType = LicenseDescription('FreeType',
            'FreeType License')
    GPLv2Plus = LicenseDescription('GPL-2+',
            'GNU General Public License, version 2 or later')
    GPLv3Plus = LicenseDescription('GPL-3+',
            'GNU General Public License, version 3 or later')
    LGPLv2Plus = LicenseDescription('LGPL-2+',
            'GNU Lesser General Public License, version 2 or later')
    LGPLv2_1Plus = LicenseDescription('LGPL-2.1+',
            'GNU Lesser General Public License, version 2.1 or later')
    LGPLv3Plus = LicenseDescription('LGPL-3+',
            'GNU Lesser General Public License, version 3 or later')
    LibPNG = LicenseDescription('LibPNG',
            'LibPNG License')
    MPLv1_1 = LicenseDescription('MPL-1.1',
            'Mozilla Public License Version 1.1')
    MPLv2 = LicenseDescription('MPL-2',
            'Mozilla Public License Version 2.0')
    MIT = LicenseDescription('MIT',
            'MIT License')
    OPENSSL = LicenseDescription('OpenSSL',
            'OpenSSL License')
    Proprietary = LicenseDescription('Proprietary',
            'Proprietary License')
    Misc = LicenseDescription('Misc',
            'Miscellaneous license information')

class LibraryType:
    NONE = 'none'
    STATIC = 'static'
    SHARED = 'shared'
    BOTH = 'both'
<|endoftext|>


<commit_before>#!/usr/bin/env python3
""" This file is a sample implementation of OAuth with OK """
import urllib.parse

from flask import Flask, redirect, url_for, session, request, jsonify, abort
from flask_oauthlib.client import OAuth
import requests

def create_client(app):
    oauth = OAuth(app)

    remote = oauth.remote_app(
        'dev',  # Client Name
        consumer_key='normal',
        consumer_secret='normal',
        request_token_params={'scope': 'all'},
        base_url='http://localhost:5000/api/v3/',
        request_token_url=None,
        access_token_method='POST',
        access_token_url='http://localhost:5000/oauth/token',
        authorize_url='http://localhost:5000/oauth/authorize'
    )

    # def check_req(uri, headers, body):
    #     """ Add access_token to the URL Request. """
    #     if 'access_token' not in uri and session.get('dev_token'):
    #         params = {'access_token': session.get('dev_token')[0]}
    #         url_parts = list(urllib.parse.urlparse(uri))
    #         query = dict(urllib.parse.parse_qsl(url_parts[4]))
    #         query.update(params)
    #
    #         url_parts[4] = urllib.parse.urlencode(query)
    #         uri = urllib.parse.urlunparse(url_parts)
    #     return uri, headers, body
    # remote.pre_request = check_req

    @app.route('/')
    def index():
        if 'dev_token' in session:
            ret = remote.get('user', token=session['dev_token'])
            return jsonify(ret.data)
            return "Welcome {}".format(ret.data['data']['email'])
        return redirect(url_for('login'))

    @app.route('/login')
    def login():
        return remote.authorize(callback=url_for('authorized', _external=True))

    @app.route('/logout')
    def logout():
        session.pop('dev_token', None)
        return redirect(url_for('index'))

    @app.route('/authorized')
    def authorized():
        resp = remote.authorized_response()
        if resp is None:
            return 'Access denied: error=%s' % (
                request.args['error']
            )
        if isinstance(resp, dict) and 'access_token' in resp:
            session['dev_token'] = (resp['access_token'], '')
            return jsonify(resp)
        return str(resp)

    @app.route('/user')
    def client_method():
        token = session['dev_token'][0]
        r = requests.get('http://localhost:5000/api/v3/user/?access_token={}'.format(token))
        r.raise_for_status()
        return jsonify(r.json())

    @remote.tokengetter
    def get_oauth_token():
        return session.get('dev_token')

    return remote

if __name__ == '__main__':
    import os
    os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = 'true'
    # DEBUG=1 python oauth2_client.py
    app = Flask(__name__)
    app.debug = True
    app.secret_key = 'development'
    create_client(app)
    app.run(host='127.0.0.1', port=8000)
<commit_msg>Use correct server name in example<commit_after>#!/usr/bin/env python3
""" This file is a sample implementation of OAuth with OK """
import urllib.parse

from flask import Flask, redirect, url_for, session, request, jsonify, abort
from flask_oauthlib.client import OAuth
import requests

def create_client(app):
    oauth = OAuth(app)

    remote = oauth.remote_app(
        'ok-server',  # Server Name
        consumer_key='normal',
        consumer_secret='normal',
        request_token_params={'scope': 'all'},
        base_url='http://localhost:5000/api/v3/',
        request_token_url=None,
        access_token_method='POST',
        access_token_url='http://localhost:5000/oauth/token',
        authorize_url='http://localhost:5000/oauth/authorize'
    )

    # def check_req(uri, headers, body):
    #     """ Add access_token to the URL Request. """
    #     if 'access_token' not in uri and session.get('dev_token'):
    #         params = {'access_token': session.get('dev_token')[0]}
    #         url_parts = list(urllib.parse.urlparse(uri))
    #         query = dict(urllib.parse.parse_qsl(url_parts[4]))
    #         query.update(params)
    #
    #         url_parts[4] = urllib.parse.urlencode(query)
    #         uri = urllib.parse.urlunparse(url_parts)
    #     return uri, headers, body
    # remote.pre_request = check_req

    @app.route('/')
    def index():
        if 'dev_token' in session:
            ret = remote.get('user', token=session['dev_token'])
            return jsonify(ret.data)
            return "Welcome {}".format(ret.data['data']['email'])
        return redirect(url_for('login'))

    @app.route('/login')
    def login():
        return remote.authorize(callback=url_for('authorized', _external=True))

    @app.route('/logout')
    def logout():
        session.pop('dev_token', None)
        return redirect(url_for('index'))

    @app.route('/authorized')
    def authorized():
        resp = remote.authorized_response()
        if resp is None:
            return 'Access denied: error=%s' % (
                request.args['error']
            )
        if isinstance(resp, dict) and 'access_token' in resp:
            session['dev_token'] = (resp['access_token'], '')
            return jsonify(resp)
        return str(resp)

    @app.route('/user')
    def client_method():
        token = session['dev_token'][0]
        r = requests.get('http://localhost:5000/api/v3/user/?access_token={}'.format(token))
        r.raise_for_status()
        return jsonify(r.json())

    @remote.tokengetter
    def get_oauth_token():
        return session.get('dev_token')

    return remote

if __name__ == '__main__':
    import os
    os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = 'true'
    # DEBUG=1 python oauth2_client.py
    app = Flask(__name__)
    app.debug = True
    app.secret_key = 'development'
    create_client(app)
    app.run(host='127.0.0.1', port=8000)
<|endoftext|>


<commit_before>#!/usr/bin/python

import argparse
import os
import struct
import sys

TAG_NAMES = {
  1:"Utf8",
  3:"Integer",
  4:"Float",
  5:"Long",
  6:"Double",
  7:"Class",
  8:"String",
  9:"Fieldref",
  10:"Methodref",
  11:"InterfaceMethodref",
  12:"NameAndType",
  15:"MethodHandle",
  16:"MethodType",
  18:"InvokeDynamic"
}

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def readToNextCpStruct(clazz):
  tag = ord(clazz.read(1))

  # print "Tag %d %s" % (tag, TAG_NAMES[tag])
  res = TAG_NAMES[tag]

  remainingSeek = 0

  if tag == 1:
    # FIXME: Yes, this will blow badly on encountering anything other than ASCII
    # https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html
    strLen = readU2(clazz)
    res += " "
    res += clazz.read(strLen)
    # print struct.unpack("s", clazz.read(strLen))[0]
    # remainingSeek = readU2(clazz)
  elif tag == 7:
    remainingSeek = 2
  elif tag == 10:
    remainingSeek = 4
  elif tag == 12:
    remainingSeek = 4
  else:
    print "ERROR: Unrecognized tag %d" % tag
    sys.exit(1)

  # print "Remaining seek %d" % remainingSeek

  if remainingSeek > 0:
    clazz.seek(remainingSeek, os.SEEK_CUR)

  return res

def readU2(clazz):
  return struct.unpack(">H", clazz.read(2))[0]

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as clazz:
  clazz.seek(8)
  cpCount = readU2(clazz) - 1
  cpIndex = 1

  print "Constant pool count: %d" % cpCount;

  while cpIndex <= cpCount:
    # print "Reading field %d" % cpIndex
    print "%d %s" % (cpIndex, readToNextCpStruct(clazz))
    cpIndex += 1

  # Skip access flags, tjhis class, super class refs
  clazz.seek(6, os.SEEK_CUR)

  # TODO: Yes, need to deal with when these aren't actually 0!
  print "\nInterfaces count: %d" % readU2(clazz)
  print "Fields count: %d" % readU2(clazz)
  print "Methods count: %d" % readU2(clazz)

  print "Pos: %x" % clazz.tell()
<commit_msg>Add remaining size of tags 8 and 9<commit_after>#!/usr/bin/python

import argparse
import os
import struct
import sys

TAG_NAMES = {
  1:"Utf8",
  3:"Integer",
  4:"Float",
  5:"Long",
  6:"Double",
  7:"Class",
  8:"String",
  9:"Fieldref",
  10:"Methodref",
  11:"InterfaceMethodref",
  12:"NameAndType",
  15:"MethodHandle",
  16:"MethodType",
  18:"InvokeDynamic"
}

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def readToNextCpStruct(clazz):
  tag = ord(clazz.read(1))

  # print "Tag %d %s" % (tag, TAG_NAMES[tag])
  res = TAG_NAMES[tag]

  remainingSeek = 0

  if tag == 1:
    # FIXME: Yes, this will blow badly on encountering anything other than ASCII
    # https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html
    strLen = readU2(clazz)
    res += " "
    res += clazz.read(strLen)
    # print struct.unpack("s", clazz.read(strLen))[0]
    # remainingSeek = readU2(clazz)
  elif tag == 7:
    remainingSeek = 2
  elif tag == 8:
    remainingSeek = 2
  elif tag == 9:
    remainingSeek = 4
  elif tag == 10:
    remainingSeek = 4
  elif tag == 12:
    remainingSeek = 4
  else:
    print "ERROR: Unrecognized tag %d" % tag
    sys.exit(1)

  # print "Remaining seek %d" % remainingSeek

  if remainingSeek > 0:
    clazz.seek(remainingSeek, os.SEEK_CUR)

  return res

def readU2(clazz):
  return struct.unpack(">H", clazz.read(2))[0]

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as clazz:
  clazz.seek(8)
  cpCount = readU2(clazz) - 1
  cpIndex = 1

  print "Constant pool count: %d" % cpCount;

  while cpIndex <= cpCount:
    # print "Reading field %d" % cpIndex
    print "%d %s" % (cpIndex, readToNextCpStruct(clazz))
    cpIndex += 1

  # Skip access flags, tjhis class, super class refs
  clazz.seek(6, os.SEEK_CUR)

  # TODO: Yes, need to deal with when these aren't actually 0!
  print "\nInterfaces count: %d" % readU2(clazz)
  print "Fields count: %d" % readU2(clazz)
  print "Methods count: %d" % readU2(clazz)

  print "Pos: %x" % clazz.tell()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#
# Copyright 2014 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import migrate
import sqlalchemy as sa

def get_alembic_version(meta):
    """Return Alembic version or None if no Alembic table exists."""
    try:
        a_ver = sa.Table(
            'alembic_version',
            meta,
            autoload=True)
        return sa.select([a_ver.c.version_num]).scalar()
    except sa.exc.NoSuchTableError:
        return None

def delete_alembic(meta):
    try:
        sa.Table(
            'alembic_version',
            meta,
            autoload=True).drop(checkfirst=True)
    except sa.exc.NoSuchTableError:
        pass

INDEXES = (
    # ([dialects], table_name, index_name, create/delete, uniq/not_uniq)
    (['mysql', 'sqlite', 'postgresql'],
     'resource',
     'resource_user_id_project_id_key',
     ('user_id', 'project_id'), True, False, True),
    (['mysql'], 'source', 'id', ('id',), False, True, False))

def index_cleanup(meta, engine_names, table_name, uniq_name, columns,
                  create, unique, limited):
    table = sa.Table(table_name, meta, autoload=True)
    if create:
        if limited and meta.bind.engine.name == 'mysql':
            # For some versions of mysql we can get an error
            # "Specified key was too long; max key length is 1000 bytes".
            # We should create an index by hand in this case with limited
            # length of columns.
            columns_mysql = ",".join((c + "(100)" for c in columns))
            sql = ("create index %s ON %s (%s)" % (uniq_name, table,
                                                   columns_mysql))
            meta.bind.engine.execute(sql)
        else:
            cols = [table.c[col] for col in columns]
            sa.Index(uniq_name, *cols, unique=unique).create()
    else:
        if unique:
            migrate.UniqueConstraint(*columns, table=table).drop()
        else:
            cols = [table.c[col] for col in columns]
            sa.Index(uniq_name, *cols, unique=unique).drop()

def change_uniq(meta, downgrade=False):
    uniq_name = 'uniq_sourceassoc0meter_id0user_id'
    columns = ('meter_id', 'user_id')

    if meta.bind.engine.name == 'sqlite':
        return

    sourceassoc = sa.Table('sourceassoc', meta, autoload=True)
    meter = sa.Table('meter', meta, autoload=True)
    user = sa.Table('user', meta, autoload=True)
    if meta.bind.engine.name == 'mysql':
        # For mysql dialect all dependent FK should be removed
        #  before renaming of constraint.
        params = {'columns': [sourceassoc.c.meter_id],
                  'refcolumns': [meter.c.id],
                  'name': 'fk_sourceassoc_meter_id'}
        migrate.ForeignKeyConstraint(**params).drop()
        params = {'columns': [sourceassoc.c.user_id],
                  'refcolumns': [user.c.id],
                  'name': 'fk_sourceassoc_user_id'}
        migrate.ForeignKeyConstraint(**params).drop()
    if downgrade:
        migrate.UniqueConstraint(*columns, table=sourceassoc,
                                 name=uniq_name).drop()
    else:
        migrate.UniqueConstraint(*columns, table=sourceassoc,
                                 name=uniq_name).create()
    if meta.bind.engine.name == 'mysql':
        params = {'columns': [sourceassoc.c.meter_id],
                  'refcolumns': [meter.c.id],
                  'name': 'fk_sourceassoc_meter_id'}
        migrate.ForeignKeyConstraint(**params).create()
        params = {'columns': [sourceassoc.c.user_id],
                  'refcolumns': [user.c.id],
                  'name': 'fk_sourceassoc_user_id'}
        migrate.ForeignKeyConstraint(**params).create()

def upgrade(migrate_engine):
    meta = sa.MetaData(bind=migrate_engine)
    a_ver = get_alembic_version(meta)

    if not a_ver:
        alarm = sa.Table('alarm', meta, autoload=True)
        repeat_act = sa.Column('repeat_actions', sa.Boolean,
                               server_default=sa.sql.expression.false())
        alarm.create_column(repeat_act)
        a_ver = '43b1a023dfaa'

    if a_ver == '43b1a023dfaa':
        meter = sa.Table('meter', meta, autoload=True)
        meter.c.resource_metadata.alter(type=sa.Text)
        a_ver = '17738166b91'

    if a_ver == '17738166b91':
        for (engine_names, table_name, uniq_name,
             columns, create, uniq, limited) in INDEXES:
            if migrate_engine in engine_names:
                index_cleanup(meta, engine_names, table_name, uniq_name,
                              columns, create, uniq, limited)
        a_ver = 'b6ae66d05e3'

    if a_ver == 'b6ae66d05e3':
        change_uniq(meta)

    delete_alembic(meta)

def downgrade(migrate_engine):
    meta = sa.MetaData(bind=migrate_engine)

    change_uniq(meta, downgrade=True)

    for (engine_names, table_name, uniq_name,
         columns, create, uniq, limited) in INDEXES:
        if migrate_engine in engine_names:
            index_cleanup(meta, engine_names, table_name, uniq_name,
                          columns, not create, uniq, limited)

    meter = sa.Table('meter', meta, autoload=True)
    meter.c.resource_metadata.alter(type=sa.String(5000))

    alarm = sa.Table('alarm', meta, autoload=True)
    repeat_act = sa.Column('repeat_actions', sa.Boolean)
    alarm.drop_column(repeat_act)
<commit_msg>028 migration script incorrectly skips over section<commit_after># -*- encoding: utf-8 -*-
#
# Copyright 2014 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import migrate
import sqlalchemy as sa

def get_alembic_version(meta):
    """Return Alembic version or None if no Alembic table exists."""
    try:
        a_ver = sa.Table(
            'alembic_version',
            meta,
            autoload=True)
        return sa.select([a_ver.c.version_num]).scalar()
    except sa.exc.NoSuchTableError:
        return None

def delete_alembic(meta):
    try:
        sa.Table(
            'alembic_version',
            meta,
            autoload=True).drop(checkfirst=True)
    except sa.exc.NoSuchTableError:
        pass

INDEXES = (
    # ([dialects], table_name, index_name, create/delete, uniq/not_uniq)
    (['mysql', 'sqlite', 'postgresql'],
     'resource',
     'resource_user_id_project_id_key',
     ('user_id', 'project_id'), True, False, True),
    (['mysql'], 'source', 'id', ('id',), False, True, False))

def index_cleanup(meta, table_name, uniq_name, columns,
                  create, unique, limited):
    table = sa.Table(table_name, meta, autoload=True)
    if create:
        if limited and meta.bind.engine.name == 'mysql':
            # For some versions of mysql we can get an error
            # "Specified key was too long; max key length is 1000 bytes".
            # We should create an index by hand in this case with limited
            # length of columns.
            columns_mysql = ",".join((c + "(100)" for c in columns))
            sql = ("create index %s ON %s (%s)" % (uniq_name, table,
                                                   columns_mysql))
            meta.bind.engine.execute(sql)
        else:
            cols = [table.c[col] for col in columns]
            sa.Index(uniq_name, *cols, unique=unique).create()
    else:
        if unique:
            migrate.UniqueConstraint(*columns, table=table,
                                     name=uniq_name).drop()
        else:
            cols = [table.c[col] for col in columns]
            sa.Index(uniq_name, *cols).drop()

def change_uniq(meta, downgrade=False):
    uniq_name = 'uniq_sourceassoc0meter_id0user_id'
    columns = ('meter_id', 'user_id')

    if meta.bind.engine.name == 'sqlite':
        return

    sourceassoc = sa.Table('sourceassoc', meta, autoload=True)
    meter = sa.Table('meter', meta, autoload=True)
    user = sa.Table('user', meta, autoload=True)
    if meta.bind.engine.name == 'mysql':
        # For mysql dialect all dependent FK should be removed
        #  before renaming of constraint.
        params = {'columns': [sourceassoc.c.meter_id],
                  'refcolumns': [meter.c.id],
                  'name': 'fk_sourceassoc_meter_id'}
        migrate.ForeignKeyConstraint(**params).drop()
        params = {'columns': [sourceassoc.c.user_id],
                  'refcolumns': [user.c.id],
                  'name': 'fk_sourceassoc_user_id'}
        migrate.ForeignKeyConstraint(**params).drop()
    if downgrade:
        migrate.UniqueConstraint(*columns, table=sourceassoc,
                                 name=uniq_name).drop()
    else:
        migrate.UniqueConstraint(*columns, table=sourceassoc,
                                 name=uniq_name).create()
    if meta.bind.engine.name == 'mysql':
        params = {'columns': [sourceassoc.c.meter_id],
                  'refcolumns': [meter.c.id],
                  'name': 'fk_sourceassoc_meter_id'}
        migrate.ForeignKeyConstraint(**params).create()
        params = {'columns': [sourceassoc.c.user_id],
                  'refcolumns': [user.c.id],
                  'name': 'fk_sourceassoc_user_id'}
        migrate.ForeignKeyConstraint(**params).create()

def upgrade(migrate_engine):
    meta = sa.MetaData(bind=migrate_engine)
    a_ver = get_alembic_version(meta)

    if not a_ver:
        alarm = sa.Table('alarm', meta, autoload=True)
        repeat_act = sa.Column('repeat_actions', sa.Boolean,
                               server_default=sa.sql.expression.false())
        alarm.create_column(repeat_act)
        a_ver = '43b1a023dfaa'

    if a_ver == '43b1a023dfaa':
        meter = sa.Table('meter', meta, autoload=True)
        meter.c.resource_metadata.alter(type=sa.Text)
        a_ver = '17738166b91'

    if a_ver == '17738166b91':
        for (engine_names, table_name, uniq_name,
             columns, create, uniq, limited) in INDEXES:
            if migrate_engine.name in engine_names:
                index_cleanup(meta, table_name, uniq_name,
                              columns, create, uniq, limited)
        a_ver = 'b6ae66d05e3'

    if a_ver == 'b6ae66d05e3':
        change_uniq(meta)

    delete_alembic(meta)

def downgrade(migrate_engine):
    meta = sa.MetaData(bind=migrate_engine)

    change_uniq(meta, downgrade=True)

    for (engine_names, table_name, uniq_name,
         columns, create, uniq, limited) in INDEXES:
        if migrate_engine.name in engine_names:
            index_cleanup(meta, table_name, uniq_name,
                          columns, not create, uniq, limited)

    meter = sa.Table('meter', meta, autoload=True)
    meter.c.resource_metadata.alter(type=sa.String(5000))

    alarm = sa.Table('alarm', meta, autoload=True)
    repeat_act = sa.Column('repeat_actions', sa.Boolean)
    alarm.drop_column(repeat_act)
<|endoftext|>


<commit_before>import csv
import os
import re
import unittest

import hgvs.parser

def gcp_file_reader(fn):
    rdr = csv.DictReader(open(fn, 'r'), delimiter='\t')
    for rec in rdr:
        if rec['id'].startswith('#'):
            continue
        yield rec

class TestParseReal(unittest.TestCase):
    longMessage = True

    def setUp(self):
        self.parser = hgvs.parser.Parser()
        self._failed = []

        infilename = 'real_gcp.tsv'
        self._infile = os.path.join(os.path.dirname(__file__), 'data', infilename)

    def test_parse_hgvsg(self):
        key = 'HGVSg'
        omissions = [r'\(\d+\)$', r'copy\d+$', r'dup\d+$']
        self._parse_all(key, omissions)

    def test_parse_hgvsc(self):
        key = 'HGVSc'
        omissions = [r'copy\d+$', r'dup\d+$']
        self._parse_all(key, omissions)

    def test_parse_hgvsp(self):
        key = 'HGVSp'
        omissions = [r'dup', r'ext', r'Met1\?']
        self._parse_all(key, omissions)

    #
    # internal methods
    #

    def _parse_all(self, key, omissions = []):
        """Parse all tags of a given type

        :param key: record key corresponding to hgvs tag type (g, c ot p)
        :type key: str
        :param omissions: list of raw string literals corresponding to unsupported types
        :type omissions: list of raw string literals
        """

        # skip anything unsupported
        regexes = [re.compile(x) for x in omissions]

        outfilename = 'gcp_real_parsed_{}.out'.format(key)
        outfile = os.path.join(os.path.dirname(__file__), 'data', outfilename)
        with open(outfile, 'w') as out:
            out.write("id\t{}\n".format(key))
            for rec in gcp_file_reader(self._infile):
                if rec[key]:
                    hits = [True for x in regexes if len(x.findall(rec[key])) > 0]
                    if len(hits) == 0:
                        try:
                            var_x = self.parser.parse_hgvs_variant(rec[key])
                        except:
                            self._append_fail(out, rec['id'], rec[key])
            msg = "# failed: {}".format(len(self._failed))
            self.assertTrue(len(self._failed) == 0, msg)

    def _append_fail(self, out, row_id, hgvs_tag):
        self._failed.append((row_id, hgvs_tag))
        out.write("{}\t{}\n".format(row_id, hgvs_tag))

if __name__ == '__main__':
    unittest.main()

## <LICENSE>
## Copyright 2014 HGVS Contributors (https://bitbucket.org/invitae/hgvs)
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
##     http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
## </LICENSE>
<commit_msg>#179: removed test_hgvs_parser_real (but kept gcp version)<commit_after><|endoftext|>


<commit_before>""" Weight functions"""

import numpy as np
from numpy import ma
from numpy import pi

from fluid.common.distance import distance

# defining some weight functions

# blackman
def _weight_blackman(r,l):
    """
    using values from wikipedia 'exact blackman'
    I'm not exactly understanding where r and l comes from
    """
    w = 0.4265 - 0.4965*np.cos(pi*r/l) + 0.0768*np.cos(2*pi*r/l)
    w[np.absolute(r)<l]=0
    return w

# triangular
def _weight_triangular_2D(x,y,l):
    """
    """
    r=(x**2+y**2)**0.5
    w=(l-r)/l
    w[r>l]=0
    return w

# hamming
def _weight_hamming(r,l):
    """
    """
    w=0.54+0.46*numpy.cos(pi*r/l)
    w[numpy.absolute(r)>l]=0
    return w

# hamming 2D
def _weight_hamming_2D(x, y, l):
    """
    """
    r = (x**2+y**2)**0.5
    w = 0.54 + 0.46*np.cos(2*pi*r/l)
    w[r>l] = 0
    return w

# hann
def _weight_hann(r,l):
    """
    """
    w=0.5*(1+np.cos(pi*r/l))
    w[np.absolute(r)>l]=0
    return w

# hann 2D
def _weight_hann_2D(x,y,l):
    """
    """
    r=(x**2+y**2)**0.5
    w=0.5*(1+np.cos(pi*r/l))
    w[r>l]=0
    return w

# lanczos 2D
def _weight_lanczos_2D(x,y,l,cutoff):
    """ Working on
    """
    #c=cutoff
    r=(x**2+y**2)**0.5
    w=np.sinc(r/cutoff)*np.sinc(r/cutoff/l)
    w[r>3*l]=0
<commit_msg>fixing the blackman function bug<commit_after>""" Weight functions"""

import numpy as np
from numpy import ma
from numpy import pi

from fluid.common.distance import distance

# defining some weight functions

# blackman
def _weight_blackman(r,l):
    """
    using values from wikipedia 'exact blackman'

    """
    w = 0.42 +  0.5*np.cos(2*pi*r/l) + 0.08*np.cos(4*pi*r/l) # fase lag-> sign change
    w[np.absolute(r)>l]=0
    return w

# triangular
def _weight_triangular_2D(x,y,l):
    """
    """
    r=(x**2+y**2)**0.5
    w=(l-r)/l
    w[r>l]=0
    return w

# hamming
def _weight_hamming(r,l):
    """
    """
    w=0.54+0.46*numpy.cos(pi*r/l)
    w[numpy.absolute(r)>l]=0
    return w

# hamming 2D
def _weight_hamming_2D(x, y, l):
    """
    """
    r = (x**2+y**2)**0.5
    w = 0.54 + 0.46*np.cos(2*pi*r/l)
    w[r>l] = 0
    return w

# hann
def _weight_hann(r,l):
    """
    """
    w=0.5*(1+np.cos(pi*r/l))
    w[np.absolute(r)>l]=0
    return w

# hann 2D
def _weight_hann_2D(x,y,l):
    """
    """
    r=(x**2+y**2)**0.5
    w=0.5*(1+np.cos(pi*r/l))
    w[r>l]=0
    return w

# lanczos 2D
def _weight_lanczos_2D(x,y,l,cutoff):
    """ Working on
    """
    #c=cutoff
    r=(x**2+y**2)**0.5
    w=np.sinc(r/cutoff)*np.sinc(r/cutoff/l)
    w[r>3*l]=0
<|endoftext|>


<commit_before>from rest_framework.views import APIView
from rest_framework.generics import ListCreateAPIView
from rest_framework.response import Response
from rest_framework import authentication, permissions
from django.contrib.auth import models
from apiv1.models import *
from apiv1.serializers import *
from django.core import serializers
from rest_framework import viewsets
from apiv1.utils import *
import json
# Create your views here.

class WebAppView(APIView):
    def get(self, request, app_name):
        try:
            marathon_client = get_marathon_client

            if app_name:
                app = request.user.webapp_set.get(name=app_name)
                app = get_info_app(app, marathon_client)
                serializer = WebAppSerializer(data=app)
            else:
                apps = request.user.webapp_set.all()
                data = []
                for app in apps:
                    app = get_info_app(app, marathon_client)
                    data.append(app)
                serializer = WebAppSerializer(data=data, many=True)
        except WebApp.DoesNotExist as e:
            data = {"status": "error", "message": "app {} does not exist".format(app_name)}
            serializer = MessageSerializer(data=data)
        except Exception as e:
            data = {"status": "error", "message": str(e)}
            serializer = MessageSerializer(data=data)

        if serializer.is_valid():
            return Response(serializer.data)
        return Response("Unserialize object!")

    def post(self, request, app_name):
        new_app = request.data
        try:
            app = WebApp()
            app_name = new_app["name"]
            if app_name.startswith(request.user.username):
                if WebApp.objects.filter(name=app_name):
                    data = {"status": "error", "message": "app name {} already existed".format(app_name)}
                else:
                    app.name = new_app["name"]
                    app.min_instances = new_app["min_instances"]
                    app.max_instances = new_app["max_instances"]
                    app.github_url = new_app["github_url"]
                    app.user = request.user
                    app.save()
                    data = {"status": "success", "message": "create app {} success".format(app_name)}
            else:
                data = {"status": "error", "message": "app name must start with your's username_"}
        except Exception as e:
            data = {"status": "error", "message": str(e)}

        serializer = MessageSerializer(data=data)
        if serializer.is_valid():
            return Response(serializer.data)
        return Response("Unserialize object!")

    def delete(self, request, app_name):
        marathon_client = get_marathon_client()
        app = WebApp.objects.filter(name=app_name).first()
        if app is None or app.user.id is not request.user.id:
            data = {"status": "error", "message": "app {} does not exist".format(app_name)}
        else:
            app.delete()
            data = {"status": "success", "message": "delete app {} success".format(app_name)}
        serializer = MessageSerializer(data=data)
        if serializer.is_valid():
            return Response(serializer.data)
        return Response("Unserialize object!")

class PolicyView(APIView):

    def get(self, request, app_name):
        try:
            app = WebApp.objects.filter(name=app_name)[0]
            policies = app.policy_set.all()
            serializer = PolicySerializer(policies, many=True)
        except IndexError as e:
            data = {"status": "error", "message": "app {} does not exist".format(app_name)}
            serializer = MessageSerializer(data=data)
            serializer.is_valid()
        return Response(serializer.data)

    def post(self, request, app_name):
        new_policy = request.data
        try:
            policy = Policy()
            for field, value in new_policy.items():
                setattr(policy,field,value)
            app = WebApp.objects.filter(name=app_name).first()
            if app is None:
                data = {"status": "error", "message": "app {} does not exist".format(app_name)}
                serializer = MessageSerializer(data=data)
            else:
                policy.web_app = app
                policy.save()
                data = {"status": "success", "message": "add policy for app {} success".format(app_name)}
        except Exception as e:
            data = {"status": "error", "message": str(e)}

        serializer = MessageSerializer(data=data)
        serializer.is_valid()
        return Response(serializer.data)
<commit_msg>edit WebAppView.post<commit_after>from rest_framework.views import APIView
from rest_framework.generics import ListCreateAPIView
from rest_framework.response import Response
from rest_framework import authentication, permissions
from django.contrib.auth import models
from apiv1.models import *
from apiv1.serializers import *
from django.core import serializers
from rest_framework import viewsets
from apiv1.utils import *
import json
# Create your views here.

class WebAppView(APIView):
    def get(self, request, app_name):
        try:
            marathon_client = get_marathon_client

            if app_name:
                app = request.user.webapp_set.get(name=app_name)
                app = get_info_app(app, marathon_client)
                serializer = WebAppSerializer(data=app)
            else:
                apps = request.user.webapp_set.all()
                data = []
                for app in apps:
                    app = get_info_app(app, marathon_client)
                    data.append(app)
                serializer = WebAppSerializer(data=data, many=True)
        except WebApp.DoesNotExist as e:
            data = {"status": "error", "message": "app {} does not exist".format(app_name)}
            serializer = MessageSerializer(data=data)
        except Exception as e:
            data = {"status": "error", "message": str(e)}
            serializer = MessageSerializer(data=data)

        if serializer.is_valid():
            return Response(serializer.data)
        return Response("Unserialize object!")

    def post(self, request, app_name):
        new_app = request.data
        try:
            app = WebApp()
            app_name = new_app["name"]
            if app_name.startswith(request.user.username):
                if WebApp.objects.filter(name=app_name):
                    data = {"status": "error", "message": "app name {} already existed".format(app_name)}
                else:
                    for field, value in new_app:
                        setattr(app,field,value)
                    app.user = request.user
                    app.save()
                    data = {"status": "success", "message": "create app {} success".format(app_name)}
            else:
                data = {"status": "error", "message": "app name must start with your's username_"}
        except Exception as e:
            data = {"status": "error", "message": str(e)}

        serializer = MessageSerializer(data=data)
        if serializer.is_valid():
            return Response(serializer.data)
        return Response("Unserialize object!")

    def delete(self, request, app_name):
        marathon_client = get_marathon_client()
        app = WebApp.objects.filter(name=app_name).first()
        if app is None or app.user.id is not request.user.id:
            data = {"status": "error", "message": "app {} does not exist".format(app_name)}
        else:
            app.delete()
            data = {"status": "success", "message": "delete app {} success".format(app_name)}
        serializer = MessageSerializer(data=data)
        if serializer.is_valid():
            return Response(serializer.data)
        return Response("Unserialize object!")

class PolicyView(APIView):

    def get(self, request, app_name):
        try:
            app = WebApp.objects.filter(name=app_name)[0]
            policies = app.policy_set.all()
            serializer = PolicySerializer(policies, many=True)
        except IndexError as e:
            data = {"status": "error", "message": "app {} does not exist".format(app_name)}
            serializer = MessageSerializer(data=data)
            serializer.is_valid()
        return Response(serializer.data)

    def post(self, request, app_name):
        new_policy = request.data
        try:
            policy = Policy()
            for field, value in new_policy.items():
                setattr(policy,field,value)
            app = WebApp.objects.filter(name=app_name).first()
            if app is None:
                data = {"status": "error", "message": "app {} does not exist".format(app_name)}
                serializer = MessageSerializer(data=data)
            else:
                policy.web_app = app
                policy.save()
                data = {"status": "success", "message": "add policy for app {} success".format(app_name)}
        except Exception as e:
            data = {"status": "error", "message": str(e)}

        serializer = MessageSerializer(data=data)
        serializer.is_valid()
        return Response(serializer.data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from enum import Enum

from april import Struct

class ModelType(Enum):
    dummy = 0

    song = 1
    artist = 2
    album = 3
    playlist = 4
    lyric = 5

    user = 17

class BaseModel(Struct):
    typy_ = ModelType.dummy
    _fields = ['source', 'identifier']

class BriefArtistModel(BaseModel):
    type_ = ModelType.artist
    _fields = ['name', 'cover']

class BriefAlbumModel(BaseModel):
    type_ = ModelType.album
    _fields = ['name']

class BriefSongModel(BaseModel):
    type_ = ModelType.song
    _fields = ['title', 'url', 'duration', 'brief_album', 'brief_artists']

class ArtistModel(BriefArtistModel):
    _fields = ['songs', 'desc']

    def __str__(self):
        return 'fuo://{}/artists/{}'.format(self.source, self.identifier)

class AlbumModel(BriefAlbumModel):
    _fields = ['cover', 'songs', 'artists', 'desc']

    def __str__(self):
        return 'fuo://{}/albums/{}'.format(self.source, self.identifier)

class LyricModel(BaseModel):
    type_ = ModelType.lyric
    _fields = ['song', 'content', 'trans_content']

class SongModel(BriefSongModel):
    _fields = ['album', 'artists', 'lyric', 'comments']

    @property
    def artists_name(self):
        return ','.join((artist.name for artist in self.artists))

    @property
    def album_name(self):
        return self.album.name if self.album is not None else ''

    @property
    def filename(self):
        return '{} - {}.mp3'.format(self.title, self.artists_name)

    def __str__(self):
        return 'fuo://{}/songs/{}'.format(self.source, self.identifier)  # noqa

    def __eq__(self, other):
        return all([other.source == self.source,
                    other.identifier == self.identifier])

class PlaylistModel(BaseModel):
    type_ = ModelType.playlist
    _fields = ['name', 'cover', 'songs', 'desc']

    def __str__(self):
        return 'fuo://{}/playlists/{}'.format(self.source, self.identifier)

class UserModel(BaseModel):
    type_ = ModelType.user
    _fields = ['name', 'playlists']
<commit_msg>给 Model 添加 __eq__ 方法<commit_after># -*- coding: utf-8 -*-

from enum import Enum

from april import Struct

class ModelType(Enum):
    dummy = 0

    song = 1
    artist = 2
    album = 3
    playlist = 4
    lyric = 5

    user = 17

class BaseModel(Struct):
    type_ = ModelType.dummy
    _fields = ['source', 'identifier']

    def __eq__(self, other):
        if not isinstance(other, BaseModel):
            return False
        return all([other.source == self.source,
                    other.identifier == self.identifier,
                    other.type_ == self.type_])

class BriefArtistModel(BaseModel):
    type_ = ModelType.artist
    _fields = ['name', 'cover']

class BriefAlbumModel(BaseModel):
    type_ = ModelType.album
    _fields = ['name']

class BriefSongModel(BaseModel):
    type_ = ModelType.song
    _fields = ['title', 'url', 'duration', 'brief_album', 'brief_artists']

class ArtistModel(BriefArtistModel):
    _fields = ['songs', 'desc']

    def __str__(self):
        return 'fuo://{}/artists/{}'.format(self.source, self.identifier)

class AlbumModel(BriefAlbumModel):
    _fields = ['cover', 'songs', 'artists', 'desc']

    def __str__(self):
        return 'fuo://{}/albums/{}'.format(self.source, self.identifier)

class LyricModel(BaseModel):
    type_ = ModelType.lyric
    _fields = ['song', 'content', 'trans_content']

class SongModel(BriefSongModel):
    _fields = ['album', 'artists', 'lyric', 'comments']

    @property
    def artists_name(self):
        return ','.join((artist.name for artist in self.artists))

    @property
    def album_name(self):
        return self.album.name if self.album is not None else ''

    @property
    def filename(self):
        return '{} - {}.mp3'.format(self.title, self.artists_name)

    def __str__(self):
        return 'fuo://{}/songs/{}'.format(self.source, self.identifier)  # noqa

    def __eq__(self, other):
        return all([other.source == self.source,
                    other.identifier == self.identifier])

class PlaylistModel(BaseModel):
    type_ = ModelType.playlist
    _fields = ['name', 'cover', 'songs', 'desc']

    def __str__(self):
        return 'fuo://{}/playlists/{}'.format(self.source, self.identifier)

class UserModel(BaseModel):
    type_ = ModelType.user
    _fields = ['name', 'playlists']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import queue

from chainer import Chain, ChainList, cuda, Variable
import chainer.functions as F
import chainer.links as L
from chainer.links.connection.n_step_lstm import argsort_list_descent, permutate_list
import numpy as np

class Embed(ChainList):

    def __init__(self, *args):
        embeds = []
        for i, embeddings in enumerate(args):
            vocab_size, embed_size = embeddings.shape
            embed = L.EmbedID(
                in_size=vocab_size,
                out_size=embed_size,
                initialW=embeddings,
            )
            embeds.append(embed)
        super(Embed, self).__init__(*embeds)

    def __call__(self, *xs):
        hs = []
        batch = len(xs[0])
        for i in range(batch):
            _hs = F.concat([embed(self.xp.array(_xs[i])) for _xs, embed in zip(xs, self)])
            hs.append(_hs)
        return hs

class LSTM(L.NStepLSTM):

    def __init__(self, in_size, out_size, dropout=0.5, use_cudnn=True):
        n_layers = 1
        super(LSTM, self).__init__(n_layers, in_size, out_size, dropout, use_cudnn)
        self.state_size = out_size
        self.reset_state()

    def to_cpu(self):
        super(LSTM, self).to_cpu()
        if self.cx is not None:
            self.cx.to_cpu()
        if self.hx is not None:
            self.hx.to_cpu()

    def to_gpu(self, device=None):
        super(LSTM, self).to_gpu(device)
        if self.cx is not None:
            self.cx.to_gpu(device)
        if self.hx is not None:
            self.hx.to_gpu(device)

    def set_state(self, cx, hx):
        assert isinstance(cx, Variable)
        assert isinstance(hx, Variable)
        cx_ = cx
        hx_ = hx
        if self.xp == np:
            cx_.to_cpu()
            hx_.to_cpu()
        else:
            cx_.to_gpu()
            hx_.to_gpu()
        self.cx = cx_
        self.hx = hx_

    def reset_state(self):
        self.cx = self.hx = None

    def __call__(self, xs, train=True):
        batch = len(xs)
        if self.hx is None:
            xp = self.xp
            self.hx = Variable(
                xp.zeros((self.n_layers, batch, self.state_size), dtype=xs[0].dtype),
                volatile='auto')
        if self.cx is None:
            xp = self.xp
            self.cx = Variable(
                xp.zeros((self.n_layers, batch, self.state_size), dtype=xs[0].dtype),
                volatile='auto')

        hy, cy, ys = super(LSTM, self).__call__(self.hx, self.cx, xs, train)
        self.hx, self.cx = hy, cy
        return ys

class BLSTM(Chain):

    def __init__(self, n_units, dropout=0.5):
        super(BLSTM, self).__init__(
            f_lstm=LSTM(n_units, n_units, dropout),
            b_lstm=LSTM(n_units, n_units, dropout),
        )
        self._dropout = dropout
        self._n_units = n_units

    def __call__(self, xs, train=True):
        self.f_lstm.reset_state()
        self.b_lstm.reset_state()
        xs_f = []
        xs_b = []
        for x in xs:
            xs_f.append(x)
            xs_b.append(x[::-1])
        hs_f = self.f_lstm(xs_f, train)
        hs_b = self.b_lstm(xs_b, train)
        ys = [F.concat([h_f, h_b[::-1]]) for h_f, h_b in zip(hs_f, hs_b)]
        return ys

class GlobalAttention(Chain):

    def __init__(self, n_units, score_func='general'):
        links = {}
        if score_func == 'dot':
            self._score_func = self._score_dot
        elif score_func == 'concat':
            links['W'] = L.Linear(n_units * 2, n_units * 2)
            links['v'] = L.Linear(n_units * 2, 1)
            self._score_func = self._score_concat
        else:
            links['W'] = L.Linear(n_units, n_units, nobias=True)
            self._score_func = self._score_general
        super(GlobalAttention, self).__init__(**links)

    def __call__(self, x, hs):
        batch, dim = x.shape
        alphas = 0
        _sum = 0
        for h in F.transpose_sequence(hs[:batch]):
            size = h.shape[0]
            if size < batch:
                h = F.vstack([h, Variable(self.xp.zeros((batch - size, h.shape[1]), dtype='f'))])
            score = self._score_func(x, h)
            e = F.exp(score)
            _sum += e
            alphas += F.batch_matmul(h, e)
        c = F.reshape(F.batch_matmul(F.reshape(alphas, (batch, dim)), (1 / _sum)), (batch, dim))
        return c

    def _score_general(self, x, h):
        batch, dim = x.shape
        return F.batch_matmul(F.reshape(self.W(x), (batch, 1, dim)), h)

    def _score_concat(self, x, h):
        return self.v(F.tanh(self.W(F.concat([x, h]))))

    def _score_dot(self, x, h):
        raise NotImplementedError()

class CRF(L.CRF1d):

    def __init__(self, n_label):
        super(CRF, self).__init__(n_label)

    def __call__(self, xs, ys):
        xs = permutate_list(xs, argsort_list_descent(xs), inv=False)
        xs = F.transpose_sequence(xs)
        ys = permutate_list(ys, argsort_list_descent(ys), inv=False)
        ys = F.transpose_sequence(ys)
        return super(CRF, self).__call__(xs, ys)

    def argmax(self, xs):
        xs = permutate_list(xs, argsort_list_descent(xs), inv=False)
        xs = F.transpose_sequence(xs)
        score, path = super(CRF, self).argmax(xs)
        path = F.transpose_sequence(path)
        return score, path

    def argnmax(self, xs, n=10):
        cost = cuda.to_cpu(self.cost.data)
        xs = permutate_list(xs, argsort_list_descent(xs), inv=False)
        xs = [cuda.to_cpu(x.data) for x in xs]

        scores = []
        paths = []

        for _xs in xs:
            alphas = [_xs[0]]
            for x in _xs[1:]:
                alpha = np.max(alphas[-1] + cost, axis=1) + x
                alphas.append(alpha)

            _scores = []
            _paths = []
            _end = len(_xs) - 1
            buf = n

            c = queue.PriorityQueue()
            q = queue.PriorityQueue()
            x = _xs[_end]
            for i in range(x.shape[0]):
                q.put((-alphas[_end][i], -x[i], _end, np.random.random(), np.array([i], np.int32)))
            while not q.empty() and c.qsize() < n + buf:
                beta, score, time, r, path = q.get()
                if time == 0:
                    c.put((score, r, path))
                    continue
                t = time - 1
                x = _xs[t]
                for i in range(x.shape[0]):
                    _trans = score - cost[i, path[-1]]
                    _beta = -alphas[t][i] + _trans
                    _score = _trans - x[i]
                    q.put((_beta, _score, t, np.random.random(), np.append(path, i)))
            while not c.empty() and len(_paths) < n:
                score, r, path = c.get()
                _scores.append(-score)
                _paths.append(path[::-1])
            scores.append(_scores)
            paths.append(_paths)

        return scores, paths

class CharCNN(Chain):

    def __init__(self, char_embeddings, window_size=3, dropout=0.5):
        char_vocab_size, char_embed_size = char_embeddings.shape
        super(CharCNN, self).__init__(
            embed=L.EmbedID(
                in_size=char_vocab_size,
                out_size=char_embed_size,
                initialW=char_embeddings,
            ),
            conv=L.Convolution2D(
                in_channels=1,
                out_channels=1,
                ksize=(1, window_size),
                stride=(1, 1),
                pad=(0, int(window_size / 2)),
                wscale=1,
                initialW=None,
                nobias=True,
                use_cudnn=True,
            ),
        )
        self._dropout = dropout

    def __call__(self, chars, train=True):
        if type(chars) == list:
            return F.vstack([self.forward_one(_chars, train) for _chars in chars])
        return self.forward_one(chars, train)

    def forward_one(self, chars, train=True):
        x = self.embed(self.xp.array(chars))
        x = F.dropout(x, self._dropout, train)
        h, w = x.shape
        C = self.conv(F.reshape(x, (1, 1, h, w)))
        A = F.max_pooling_2d(C, ksize=(h, 1), stride=None, pad=0, use_cudnn=True)
        y = F.reshape(A, (w,))
        return y
<commit_msg>implement FastBLSTM and Biaffine Links.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import math
import queue

from chainer import Chain, ChainList, cuda, initializers, link, Variable
import chainer.functions as F
import chainer.links as L
from chainer.links.connection.n_step_rnn import argsort_list_descent, permutate_list
import numpy as np

class Embed(ChainList):

    def __init__(self, *args):
        embeds = []
        for i, embeddings in enumerate(args):
            vocab_size, embed_size = embeddings.shape
            embed = L.EmbedID(
                in_size=vocab_size,
                out_size=embed_size,
                initialW=embeddings,
            )
            embeds.append(embed)
        super(Embed, self).__init__(*embeds)

    def __call__(self, *xs):
        hs = []
        batch = len(xs[0])
        for i in range(batch):
            _hs = F.concat([embed(self.xp.array(_xs[i])) for _xs, embed in zip(xs, self)])
            hs.append(_hs)
        return hs

class LSTM(L.NStepLSTM):

    def __init__(self, in_size, out_size, dropout=0.5, use_cudnn=True):
        n_layers = 1
        super(LSTM, self).__init__(n_layers, in_size, out_size, dropout, use_cudnn)
        self.state_size = out_size
        self.reset_state()

    def to_cpu(self):
        super(LSTM, self).to_cpu()
        if self.cx is not None:
            self.cx.to_cpu()
        if self.hx is not None:
            self.hx.to_cpu()

    def to_gpu(self, device=None):
        super(LSTM, self).to_gpu(device)
        if self.cx is not None:
            self.cx.to_gpu(device)
        if self.hx is not None:
            self.hx.to_gpu(device)

    def set_state(self, cx, hx):
        assert isinstance(cx, Variable)
        assert isinstance(hx, Variable)
        cx_ = cx
        hx_ = hx
        if self.xp == np:
            cx_.to_cpu()
            hx_.to_cpu()
        else:
            cx_.to_gpu()
            hx_.to_gpu()
        self.cx = cx_
        self.hx = hx_

    def reset_state(self):
        self.cx = self.hx = None

    def __call__(self, xs, train=True):
        batch = len(xs)
        if self.hx is None:
            xp = self.xp
            self.hx = Variable(
                xp.zeros((self.n_layers, batch, self.state_size), dtype=xs[0].dtype),
                volatile='auto')
        if self.cx is None:
            xp = self.xp
            self.cx = Variable(
                xp.zeros((self.n_layers, batch, self.state_size), dtype=xs[0].dtype),
                volatile='auto')

        hy, cy, ys = super(LSTM, self).__call__(self.hx, self.cx, xs, train)
        self.hx, self.cx = hy, cy
        return ys

class BLSTM(Chain):
    """@TODO: implement same interface as FastBLSTM"""

    def __init__(self, n_units, dropout=0.5):
        super(BLSTM, self).__init__(
            f_lstm=LSTM(n_units, n_units, dropout),
            b_lstm=LSTM(n_units, n_units, dropout),
        )
        self._dropout = dropout
        self._n_units = n_units

    def __call__(self, xs, train=True):
        self.f_lstm.reset_state()
        self.b_lstm.reset_state()
        xs_f = []
        xs_b = []
        for x in xs:
            xs_f.append(x)
            xs_b.append(x[::-1])
        hs_f = self.f_lstm(xs_f, train)
        hs_b = self.b_lstm(xs_b, train)
        ys = [F.concat([h_f, h_b[::-1]]) for h_f, h_b in zip(hs_f, hs_b)]
        return ys

class FastBLSTM(L.NStepBiLSTM):

    def __init__(self, n_layers, in_size, out_size, dropout=0.5, use_cudnn=True):
        super(FastBLSTM, self).__init__(n_layers, in_size, out_size, dropout, use_cudnn)

    def __call__(self, xs, train=True):
        hx, cx = None, None
        hy, cy, ys = super(FastBLSTM, self).__call__(hx, cx, xs, train)
        return ys

class GlobalAttention(Chain):

    def __init__(self, n_units, score_func='general'):
        links = {}
        if score_func == 'dot':
            self._score_func = self._score_dot
        elif score_func == 'concat':
            links['W'] = L.Linear(n_units * 2, n_units * 2)
            links['v'] = L.Linear(n_units * 2, 1)
            self._score_func = self._score_concat
        else:
            links['W'] = L.Linear(n_units, n_units, nobias=True)
            self._score_func = self._score_general
        super(GlobalAttention, self).__init__(**links)

    def __call__(self, x, hs):
        batch, dim = x.shape
        alphas = 0
        _sum = 0
        for h in F.transpose_sequence(hs[:batch]):
            size = h.shape[0]
            if size < batch:
                h = F.vstack([h, Variable(self.xp.zeros((batch - size, h.shape[1]), dtype='f'))])
            score = self._score_func(x, h)
            e = F.exp(score)
            _sum += e
            alphas += F.batch_matmul(h, e)
        c = F.reshape(F.batch_matmul(F.reshape(alphas, (batch, dim)), (1 / _sum)), (batch, dim))
        return c

    def _score_general(self, x, h):
        batch, dim = x.shape
        return F.batch_matmul(F.reshape(self.W(x), (batch, 1, dim)), h)

    def _score_concat(self, x, h):
        return self.v(F.tanh(self.W(F.concat([x, h]))))

    def _score_dot(self, x, h):
        raise NotImplementedError()

class CRF(L.CRF1d):

    def __init__(self, n_label):
        super(CRF, self).__init__(n_label)

    def __call__(self, xs, ys):
        xs = permutate_list(xs, argsort_list_descent(xs), inv=False)
        xs = F.transpose_sequence(xs)
        ys = permutate_list(ys, argsort_list_descent(ys), inv=False)
        ys = F.transpose_sequence(ys)
        return super(CRF, self).__call__(xs, ys)

    def argmax(self, xs):
        xs = permutate_list(xs, argsort_list_descent(xs), inv=False)
        xs = F.transpose_sequence(xs)
        score, path = super(CRF, self).argmax(xs)
        path = F.transpose_sequence(path)
        return score, path

    def argnmax(self, xs, n=10):
        cost = cuda.to_cpu(self.cost.data)
        xs = permutate_list(xs, argsort_list_descent(xs), inv=False)
        xs = [cuda.to_cpu(x.data) for x in xs]

        scores = []
        paths = []

        for _xs in xs:
            alphas = [_xs[0]]
            for x in _xs[1:]:
                alpha = np.max(alphas[-1] + cost, axis=1) + x
                alphas.append(alpha)

            _scores = []
            _paths = []
            _end = len(_xs) - 1
            buf = n

            c = queue.PriorityQueue()
            q = queue.PriorityQueue()
            x = _xs[_end]
            for i in range(x.shape[0]):
                q.put((-alphas[_end][i], -x[i], _end, np.random.random(), np.array([i], np.int32)))
            while not q.empty() and c.qsize() < n + buf:
                beta, score, time, r, path = q.get()
                if time == 0:
                    c.put((score, r, path))
                    continue
                t = time - 1
                x = _xs[t]
                for i in range(x.shape[0]):
                    _trans = score - cost[i, path[-1]]
                    _beta = -alphas[t][i] + _trans
                    _score = _trans - x[i]
                    q.put((_beta, _score, t, np.random.random(), np.append(path, i)))
            while not c.empty() and len(_paths) < n:
                score, r, path = c.get()
                _scores.append(-score)
                _paths.append(path[::-1])
            scores.append(_scores)
            paths.append(_paths)

        return scores, paths

class CharCNN(Chain):

    def __init__(self, char_embeddings, window_size=3, dropout=0.5):
        char_vocab_size, char_embed_size = char_embeddings.shape
        super(CharCNN, self).__init__(
            embed=L.EmbedID(
                in_size=char_vocab_size,
                out_size=char_embed_size,
                initialW=char_embeddings,
            ),
            conv=L.Convolution2D(
                in_channels=1,
                out_channels=1,
                ksize=(1, window_size),
                stride=(1, 1),
                pad=(0, int(window_size / 2)),
                wscale=1,
                initialW=None,
                nobias=True,
                use_cudnn=True,
            ),
        )
        self._dropout = dropout

    def __call__(self, chars, train=True):
        if type(chars) == list:
            return F.vstack([self.forward_one(_chars, train) for _chars in chars])
        return self.forward_one(chars, train)

    def forward_one(self, chars, train=True):
        x = self.embed(self.xp.array(chars))
        x = F.dropout(x, self._dropout, train)
        h, w = x.shape
        C = self.conv(F.reshape(x, (1, 1, h, w)))
        A = F.max_pooling_2d(C, ksize=(h, 1), stride=None, pad=0, use_cudnn=True)
        y = F.reshape(A, (w,))
        return y

class Biaffine(link.Link):

    def __init__(self, in_size, out_size, wscale=1, initialW=None):
        super(Biaffine, self).__init__()
        self.initialW = initialW
        self.wscale = wscale
        self.out_size = out_size
        self._W_initializer = initializers._get_initializer(
            initialW, math.sqrt(wscale))
        self._initialize_params(in_size)

    def _initialize_params(self, in_size):
        self.add_param('W', (in_size + 1, self.out_size),
                       initializer=self._W_initializer)

    def __call__(self, x1, x2):
        """https://github.com/tdozat/Parser/blob/master/lib/linalg.py"""
        dim = len(x1.shape)
        if dim == 3:
            return self.forward_batch(x1, x2)
        elif dim == 2:
            return self.forward_one(x1, x2)
        else:
            raise RuntimeError()

    def forward_one(self, x1, x2):
        xp = cuda.get_array_module(x1.data)
        l, d = x2.shape
        return F.matmul(F.concat([x1, xp.ones((l, 1), 'f')]), F.matmul(self.W, x2, transb=True))

    def forward_batch(self, x1, x2):
        xp = cuda.get_array_module(x1.data)
        b, l, d = x2.shape
        return F.batch_matmul(F.concat([x1, xp.ones((b, l, 1), 'f')], 2),
                              F.reshape(F.linear(F.reshape(x2, (b * l, -1)), self.W), (b, l, -1)), transb=True)
<|endoftext|>


<commit_before>#Import Libraries
import time             #A collection of time related commands
import RPi.GPIO as GPIO #The GPIO commands

#Set the GPIO pin naming mode
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)

#Set pins 18, 23 and 24 to be output
GPIO.setup(18,GPIO.OUT)
GPIO.setup(23,GPIO.OUT)
GPIO.setup(24,GPIO.OUT)

#Turn LEDs on
GPIO.output(18,GPIO.HIGH)
GPIO.output(23,GPIO.HIGH)
GPIO.output(24,GPIO.HIGH)

time.sleep(1) #Pause for 1 second

#Turn LEDs off
GPIO.output(18,GPIO.LOW)
GPIO.output(23,GPIO.LOW)
GPIO.output(24,GPIO.LOW)

time.sleep(1) #Pause for 1 second

#Turn LEDs on
GPIO.output(18,GPIO.HIGH)
GPIO.output(23,GPIO.HIGH)
GPIO.output(24,GPIO.HIGH)

time.sleep(1) #Pause for 1 second

#Turn LEDs off
GPIO.output(18,GPIO.LOW)
GPIO.output(23,GPIO.LOW)
GPIO.output(24,GPIO.LOW)

#Clean up the GPIO pins
GPIO.cleanup()
<commit_msg>Updated to python3<commit_after># Import Libraries
import time             # A collection of time related commands
import RPi.GPIO as GPIO # The GPIO commands

# Set the GPIO pin naming mode
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)

# Set pins 18, 23 and 24 to be output
GPIO.setup(18, GPIO.OUT)
GPIO.setup(23, GPIO.OUT)
GPIO.setup(24, GPIO.OUT)

# Turn LEDs on
GPIO.output(18, GPIO.HIGH)
GPIO.output(23, GPIO.HIGH)
GPIO.output(24, GPIO.HIGH)

time.sleep(1) # Pause for 1 second

# Turn LEDs off
GPIO.output(18, GPIO.LOW)
GPIO.output(23, GPIO.LOW)
GPIO.output(24, GPIO.LOW)

time.sleep(1) # Pause for 1 second

# Turn LEDs on
GPIO.output(18, GPIO.HIGH)
GPIO.output(23, GPIO.HIGH)
GPIO.output(24, GPIO.HIGH)

time.sleep(1) # Pause for 1 second

# Turn LEDs off
GPIO.output(18, GPIO.LOW)
GPIO.output(23, GPIO.LOW)
GPIO.output(24, GPIO.LOW)

# Clean up the GPIO pins
GPIO.cleanup()
<|endoftext|>


<commit_before>import os
import struct
import sys

from twisted.internet import defer, endpoints, protocol, task
from twisted import logger

from wip.common import describe_socket, READY_BYTE

class AlwaysAbortFactory(protocol.Factory):
    log = logger.Logger()

    def buildProtocol(self, addr):
        self.log.warn('rejecting incoming connection: {addr}', addr=addr)
        return None

class HandoffProtocol(protocol.Protocol):
    done = False

    def dataReceived(self, datum):
        if self.done or datum != READY_BYTE:
            return
        self.transport.write(self.factory.handoff_data)
        self.transport.sendFileDescriptor(self.factory.handoff_fd)
        self.transport.loseConnection()
        self.done = True

@defer.inlineCallbacks
def main(reactor, server_endpoint_string, handoff_endpoint_string):
    logger.globalLogBeginner.beginLoggingTo(
        [logger.textFileLogObserver(sys.stderr)])
    server_endpoint = endpoints.serverFromString(
        reactor, server_endpoint_string)
    server_port = yield server_endpoint.listen(AlwaysAbortFactory())
    handoff_factory = protocol.Factory.forProtocol(HandoffProtocol)
    handoff_factory.handoff_fd = os.dup(server_port.fileno())
    handoff_factory.handoff_data = describe_socket(server_port.socket)
    reactor.removeReader(server_port)

    handoff_endpoint = endpoints.serverFromString(
        reactor, handoff_endpoint_string)
    yield handoff_endpoint.listen(handoff_factory)
    yield defer.Deferred()

if __name__ == '__main__':
    task.react(main, sys.argv[1:])
<commit_msg>nuke unused import<commit_after>import os
import sys

from twisted.internet import defer, endpoints, protocol, task
from twisted import logger

from wip.common import describe_socket, READY_BYTE

class AlwaysAbortFactory(protocol.Factory):
    log = logger.Logger()

    def buildProtocol(self, addr):
        self.log.warn('rejecting incoming connection: {addr}', addr=addr)
        return None

class HandoffProtocol(protocol.Protocol):
    done = False

    def dataReceived(self, datum):
        if self.done or datum != READY_BYTE:
            return
        self.transport.write(self.factory.handoff_data)
        self.transport.sendFileDescriptor(self.factory.handoff_fd)
        self.transport.loseConnection()
        self.done = True

@defer.inlineCallbacks
def main(reactor, server_endpoint_string, handoff_endpoint_string):
    logger.globalLogBeginner.beginLoggingTo(
        [logger.textFileLogObserver(sys.stderr)])
    server_endpoint = endpoints.serverFromString(
        reactor, server_endpoint_string)
    server_port = yield server_endpoint.listen(AlwaysAbortFactory())
    handoff_factory = protocol.Factory.forProtocol(HandoffProtocol)
    handoff_factory.handoff_fd = os.dup(server_port.fileno())
    handoff_factory.handoff_data = describe_socket(server_port.socket)
    reactor.removeReader(server_port)

    handoff_endpoint = endpoints.serverFromString(
        reactor, handoff_endpoint_string)
    yield handoff_endpoint.listen(handoff_factory)
    yield defer.Deferred()

if __name__ == '__main__':
    task.react(main, sys.argv[1:])
<|endoftext|>