<commit_before># Maildir repository support
# Copyright (C) 2002 John Goerzen
# <jgoerzen@complete.org>
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA

from Base import BaseRepository
from offlineimap import folder
from offlineimap.ui import getglobalui
from offlineimap.error import OfflineImapError
import os
from stat import *

class MaildirRepository(BaseRepository):
    def __init__(self, reposname, account):
        """Initialize a MaildirRepository object.  Takes a path name
        to the directory holding all the Maildir directories."""
        BaseRepository.__init__(self, reposname, account)

        self.root = self.getlocalroot()
        self.folders = None
        self.ui = getglobalui()
        self.debug("MaildirRepository initialized, sep is " + repr(self.getsep()))
        self.folder_atimes = []

        # Create the top-level folder if it doesn't exist
        if not os.path.isdir(self.root):
            os.mkdir(self.root, 0700)

    def _append_folder_atimes(self, foldername):
        """Store the atimes of a folder's new|cur in self.folder_atimes"""
        p = os.path.join(self.root, foldername)
        new = os.path.join(p, 'new')
        cur = os.path.join(p, 'cur')
        atimes = (p, os.path.getatime(new), os.path.getatime(cur))
        self.folder_atimes.append(atimes)

    def restore_atime(self):
        """Sets folders' atime back to their values after a sync

        Controlled by the 'restoreatime' config parameter."""
        if not self.getconfboolean('restoreatime', False):
            return # not configured to restore

        for (dirpath, new_atime, cur_atime) in self.folder_atimes:
            new_dir = os.path.join(dirpath, 'new')
            cur_dir = os.path.join(dirpath, 'cur')
            os.utime(new_dir, (new_atime, os.path.getmtime(new_dir)))
            os.utime(cur_dir, (cur_atime, os.path.getmtime(cur_dir)))

    def getlocalroot(self):
        return os.path.expanduser(self.getconf('localfolders'))

    def debug(self, msg):
        self.ui.debug('maildir', msg)

    def getsep(self):
        return self.getconf('sep', '.').strip()

    def makefolder(self, foldername):
        """Create new Maildir folder if necessary

        This will not update the list cached in getfolders(). You will
        need to invoke :meth:`forgetfolders` to force new caching when
        you are done creating folders yourself.

        :param foldername: A relative mailbox name. The maildir will be
            created in self.root+'/'+foldername. All intermediate folder
            levels will be created if they do not exist yet. 'cur',
            'tmp', and 'new' subfolders will be created in the maildir.
        """
        self.debug("makefolder called with arg '%s'" % (foldername))
        full_path = os.path.abspath(os.path.join(self.root, foldername))

        # sanity tests
        if self.getsep() == '/':
            for component in foldername.split('/'):
                assert not component in ['new', 'cur', 'tmp'],\
                    "When using nested folders (/ as a Maildir separator), "\
                    "folder names may not contain 'new', 'cur', 'tmp'."
        assert foldername.find('../') == -1, "Folder names may not contain ../"
        assert not foldername.startswith('/'), "Folder names may not begin with /"

        # If we're using hierarchical folders, it's possible that
        # sub-folders may be created before higher-up ones.
        self.debug("makefolder: calling makedirs '%s'" % full_path)
        try:
            os.makedirs(full_path, 0700)
        except OSError, e:
            if e.errno == 17 and os.path.isdir(full_path):
                self.debug("makefolder: '%s' already a directory" % foldername)
            else:
                raise
        for subdir in ['cur', 'new', 'tmp']:
            try:
                os.mkdir(os.path.join(full_path, subdir), 0700)
            except OSError, e:
                if e.errno == 17 and os.path.isdir(full_path):
                    self.debug("makefolder: '%s' already has subdir %s" %
                               (foldername, subdir))
                else:
                    raise

    def deletefolder(self, foldername):
        self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername)

    def getfolder(self, foldername):
        """Return a Folder instance of this Maildir

        If necessary, scan and cache all foldernames to make sure that
        we only return existing folders and that 2 calls with the same
        name will return the same object."""
        # getfolders() will scan and cache the values *if* necessary
        folders = self.getfolders()
        for folder in folders:
            if foldername == folder.name:
                return folder
        raise OfflineImapError("getfolder() asked for a nonexisting "
                               "folder '%s'." % foldername,
                               OfflineImapError.ERROR.FOLDER)

    def _getfolders_scandir(self, root, extension = None):
        """Recursively scan folder 'root'; return a list of MailDirFolder

        :param root: (absolute) path to Maildir root
        :param extension: (relative) subfolder to examine within root"""
        self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s" \
                   % (root, extension))
        retval = []

        # Configure the full path to this repository -- "toppath"
        if extension:
            toppath = os.path.join(root, extension)
        else:
            toppath = root
        self.debug("  toppath = %s" % toppath)

        # Iterate over directories in top & top itself.
        for dirname in os.listdir(toppath) + ['']:
            self.debug("  dirname = %s" % dirname)
            if dirname in ['cur', 'new', 'tmp']:
                self.debug("  skipping this dir (Maildir special)")
                # Bypass special files.
                continue
            fullname = os.path.join(toppath, dirname)
            if not os.path.isdir(fullname):
                self.debug("  skipping this entry (not a directory)")
                # Not a directory -- not a folder.
                continue
            foldername = dirname
            if extension and dirname != '':
                foldername = os.path.join(extension, dirname)
            if (os.path.isdir(os.path.join(fullname, 'cur')) and
                os.path.isdir(os.path.join(fullname, 'new')) and
                os.path.isdir(os.path.join(fullname, 'tmp'))):
                # This directory has maildir stuff -- process
                self.debug("  This is maildir folder '%s'." % foldername)
                if self.getconfboolean('restoreatime', False):
                    self._append_folder_atimes(foldername)
                retval.append(folder.Maildir.MaildirFolder(self.root,
                                                           foldername,
                                                           self.getsep(),
                                                           self))
                # filter out the folder?
                if not self.folderfilter(foldername):
                    self.debug("Filtering out '%s'[%s] due to folderfilt"
                               "er" % (foldername, self))
                    retval[-1].sync_this = False

            if self.getsep() == '/' and dirname != '':
                # Recursively check sub-directories for folders too.
                retval.extend(self._getfolders_scandir(root, foldername))
        self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \
                   repr([x.getname() for x in retval]))
        return retval

    def getfolders(self):
        if self.folders == None:
            self.folders = self._getfolders_scandir(self.root)
        return self.folders

    def forgetfolders(self):
        """Forgets the cached list of folders, if any.  Useful to run
        after a sync run."""
        self.folders = None
<commit_msg>Folder could not be created<commit_after># Maildir repository support
# Copyright (C) 2002 John Goerzen
# <jgoerzen@complete.org>
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA

from Base import BaseRepository
from offlineimap import folder
from offlineimap.ui import getglobalui
from offlineimap.error import OfflineImapError
import os
from stat import *

class MaildirRepository(BaseRepository):
    def __init__(self, reposname, account):
        """Initialize a MaildirRepository object.  Takes a path name
        to the directory holding all the Maildir directories."""
        BaseRepository.__init__(self, reposname, account)

        self.root = self.getlocalroot()
        self.folders = None
        self.ui = getglobalui()
        self.debug("MaildirRepository initialized, sep is " + repr(self.getsep()))
        self.folder_atimes = []

        # Create the top-level folder if it doesn't exist
        if not os.path.isdir(self.root):
            os.mkdir(self.root, 0700)

    def _append_folder_atimes(self, foldername):
        """Store the atimes of a folder's new|cur in self.folder_atimes"""
        p = os.path.join(self.root, foldername)
        new = os.path.join(p, 'new')
        cur = os.path.join(p, 'cur')
        atimes = (p, os.path.getatime(new), os.path.getatime(cur))
        self.folder_atimes.append(atimes)

    def restore_atime(self):
        """Sets folders' atime back to their values after a sync

        Controlled by the 'restoreatime' config parameter."""
        if not self.getconfboolean('restoreatime', False):
            return # not configured to restore

        for (dirpath, new_atime, cur_atime) in self.folder_atimes:
            new_dir = os.path.join(dirpath, 'new')
            cur_dir = os.path.join(dirpath, 'cur')
            os.utime(new_dir, (new_atime, os.path.getmtime(new_dir)))
            os.utime(cur_dir, (cur_atime, os.path.getmtime(cur_dir)))

    def getlocalroot(self):
        return os.path.expanduser(self.getconf('localfolders'))

    def debug(self, msg):
        self.ui.debug('maildir', msg)

    def getsep(self):
        return self.getconf('sep', '.').strip()

    def makefolder(self, foldername):
        """Create new Maildir folder if necessary

        This will not update the list cached in getfolders(). You will
        need to invoke :meth:`forgetfolders` to force new caching when
        you are done creating folders yourself.

        :param foldername: A relative mailbox name. The maildir will be
            created in self.root+'/'+foldername. All intermediate folder
            levels will be created if they do not exist yet. 'cur',
            'tmp', and 'new' subfolders will be created in the maildir.
        """
        self.debug("makefolder called with arg '%s'" % (foldername))
        full_path = os.path.abspath(os.path.join(self.root, foldername))

        # sanity tests
        if self.getsep() == '/':
            for component in foldername.split('/'):
                assert not component in ['new', 'cur', 'tmp'],\
                    "When using nested folders (/ as a Maildir separator), "\
                    "folder names may not contain 'new', 'cur', 'tmp'."
        assert foldername.find('../') == -1, "Folder names may not contain ../"
        assert not foldername.startswith('/'), "Folder names may not begin with /"

        # If we're using hierarchical folders, it's possible that
        # sub-folders may be created before higher-up ones.
        self.debug("makefolder: calling makedirs '%s'" % full_path)
        try:
            os.makedirs(full_path, 0700)
        except OSError, e:
            if e.errno == 17 and os.path.isdir(full_path):
                self.debug("makefolder: '%s' already a directory" % foldername)
            else:
                raise
        for subdir in ['cur', 'new', 'tmp']:
            try:
                os.mkdir(os.path.join(full_path, subdir), 0700)
            except OSError, e:
                if e.errno == 17 and os.path.isdir(full_path):
                    self.debug("makefolder: '%s' already has subdir %s" %
                               (foldername, subdir))
                else:
                    raise

    def deletefolder(self, foldername):
        self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername)

    def getfolder(self, foldername):
        """Return a Folder instance of this Maildir

        If necessary, scan and cache all foldernames to make sure that
        we only return existing folders and that 2 calls with the same
        name will return the same object."""
        # getfolders() will scan and cache the values *if* necessary
        folders = self.getfolders()
        for folder in folders:
            if foldername == folder.name:
                return folder
        raise OfflineImapError("getfolder() asked for a nonexisting "
                               "folder '%s'." % foldername,
                               OfflineImapError.ERROR.FOLDER)

    def _getfolders_scandir(self, root, extension = None):
        """Recursively scan folder 'root'; return a list of MailDirFolder

        :param root: (absolute) path to Maildir root
        :param extension: (relative) subfolder to examine within root"""
        self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s" \
                   % (root, extension))
        retval = []

        # Configure the full path to this repository -- "toppath"
        if extension:
            toppath = os.path.join(root, extension)
        else:
            toppath = root
        self.debug("  toppath = %s" % toppath)

        # Iterate over directories in top & top itself.
        for dirname in os.listdir(toppath) + ['']:
            self.debug("  dirname = %s" % dirname)
            if dirname == '' and extension is not None:
                self.debug('  skip this entry (extension set)')
                continue
            if dirname in ['cur', 'new', 'tmp']:
                self.debug("  skipping this dir (Maildir special)")
                # Bypass special files.
                continue
            fullname = os.path.join(toppath, dirname)
            if not os.path.isdir(fullname):
                self.debug("  skipping this entry (not a directory)")
                # Not a directory -- not a folder.
                continue
            foldername = dirname
            if extension and dirname != '':
                foldername = os.path.join(extension, dirname)
            if (os.path.isdir(os.path.join(fullname, 'cur')) and
                os.path.isdir(os.path.join(fullname, 'new')) and
                os.path.isdir(os.path.join(fullname, 'tmp'))):
                # This directory has maildir stuff -- process
                self.debug("  This is maildir folder '%s'." % foldername)
                if self.getconfboolean('restoreatime', False):
                    self._append_folder_atimes(foldername)
                retval.append(folder.Maildir.MaildirFolder(self.root,
                                                           foldername,
                                                           self.getsep(),
                                                           self))
                # filter out the folder?
                if not self.folderfilter(foldername):
                    self.debug("Filtering out '%s'[%s] due to folderfilt"
                               "er" % (foldername, self))
                    retval[-1].sync_this = False

            if self.getsep() == '/' and dirname != '':
                # Recursively check sub-directories for folders too.
                retval.extend(self._getfolders_scandir(root, foldername))
        self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \
                   repr([x.getname() for x in retval]))
        return retval

    def getfolders(self):
        if self.folders == None:
            self.folders = self._getfolders_scandir(self.root)
        return self.folders

    def forgetfolders(self):
        """Forgets the cached list of folders, if any.  Useful to run
        after a sync run."""
        self.folders = None
<|endoftext|>


<commit_before>#!/usr/bin/python

import xmlrpclib
import json
import os
import sys

print(sys.argv)
if len(sys.argv) != 5:
    print("Must provide four arguments. {} {} {} {} {}".format(sys.argv[0],
        "LAVA_KEY", "kernel_imager", "modules_archive", "tools_commit"))
    sys.exit()

token=sys.argv[1]
kernel=sys.argv[2]
modules=sys.argv[3]
tools_commit=sys.argv[4]

job = """
{
    "health_check": false,
    "job_name": "benchmark-syscalls",
    "device_type": "x86",
    "tags": [ "dev-sda1" ],
    "timeout": 18000,
    "actions": [
        {
            "command": "boot_image"
        },
        {
            "command": "lava_command_run",
            "parameters": {
                "commands": [
                    "ifup eth0",
                    "route -n",
                    "cat /etc/resolv.conf",
                    "echo nameserver 172.18.0.12 > /etc/resolv.conf",
                    "mount /dev/sda1 /tmp",
                    "rm -rf /tmp/*"
                ]
            }
        },
        {
            "command": "lava_command_run",
            "parameters": {
                "commands": [
                    "apt-get update",
                    "apt-get install -y bsdtar psmisc wget python3 python3-pip libglib2.0-dev libffi-dev elfutils",
                    "apt-get install -y libelf-dev libmount-dev libxml2 python3-pandas python3-numpy"
                ]
            }
        },
        {
            "command": "lava_test_shell",
            "parameters": {
                "testdef_repos": [
                    {
                        "git-repo": "https://github.com/frdeso/syscall-bench-it.git",
                        "revision": "master",
                        "testdef": "lava/testcases/failing-close.yml",
                         "parameters": {}
                    },
                    {
                        "git-repo": "https://github.com/frdeso/syscall-bench-it.git",
                        "revision": "master",
                        "testdef": "lava/testcases/failing-open-efault.yml"
                    },
                    {
                        "git-repo": "https://github.com/frdeso/syscall-bench-it.git",
                        "revision": "master",
                        "testdef": "lava/testcases/failing-open-enoent.yml"
                    }
                ],
                "timeout": 18000
            }
        },
         {
            "command": "submit_results",
            "parameters": {
                "server": "http://lava-master.internal.efficios.com/RPC2/",
                "stream": "/anonymous/benchmark-kernel/"
            }
        }
    ]
}
"""
# We use the kernel image and modules archive received as argument
deploy_action={"command": "deploy_kernel",
            "parameters": {
            		"overlays": [
            		    "scp://jenkins-lava@storage.internal.efficios.com"+modules ],
                "kernel":
                "scp://jenkins-lava@storage.internal.efficios.com"+kernel,
                "nfsrootfs": "scp://jenkins-lava@storage.internal.efficios.com/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz",
                "target_type": "ubuntu"
            }
        }

# We checkout the commit id for tools
setup_action = {
            "command": "lava_command_run",
            "parameters": {
                "commands": [
                	"git clone https://github.com/frdeso/syscall-bench-it.git bm",
                    "pip3 install vlttng",
                    "vlttng --jobs=16 --profile urcu-master \
                    --profile lttng-tools-master -o \
                    projects.lttng-tools.checkout="+tools_commit+ \
                     " /tmp/virtenv"
                ]
            }
        }
job_dict= json.loads(job)
job_dict['actions'].append(deploy_action)
job_dict['actions'].append(setup_action)

username = 'frdeso'
hostname = 'lava-master.internal.efficios.com'
server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (username, token, hostname))

print server.scheduler.submit_job(json.dumps(job_dict))
<commit_msg>Fix order of lava actions<commit_after>#!/usr/bin/python

import xmlrpclib
import json
import os
import sys

print(sys.argv)
if len(sys.argv) != 5:
    print("Must provide four arguments. {} {} {} {} {}".format(sys.argv[0],
        "LAVA_KEY", "kernel_imager", "modules_archive", "tools_commit"))
    sys.exit()

token=sys.argv[1]
kernel=sys.argv[2]
modules=sys.argv[3]
tools_commit=sys.argv[4]

job = """
{
    "health_check": false,
    "job_name": "benchmark-syscalls",
    "device_type": "x86",
    "tags": [ "dev-sda1" ],
    "timeout": 18000,
    "actions": [
        {
            "command": "boot_image"
        },
        {
            "command": "lava_command_run",
            "parameters": {
                "commands": [
                    "ifup eth0",
                    "route -n",
                    "cat /etc/resolv.conf",
                    "echo nameserver 172.18.0.12 > /etc/resolv.conf",
                    "mount /dev/sda1 /tmp",
                    "rm -rf /tmp/*"
                ]
            }
        },
        {
            "command": "lava_command_run",
            "parameters": {
                "commands": [
                    "apt-get update",
                    "apt-get install -y bsdtar psmisc wget python3 python3-pip libglib2.0-dev libffi-dev elfutils",
                    "apt-get install -y libelf-dev libmount-dev libxml2 python3-pandas python3-numpy"
                ]
            }
        },
        {
            "command": "lava_test_shell",
            "parameters": {
                "testdef_repos": [
                    {
                        "git-repo": "https://github.com/frdeso/syscall-bench-it.git",
                        "revision": "master",
                        "testdef": "lava/testcases/failing-close.yml",
                         "parameters": {}
                    },
                    {
                        "git-repo": "https://github.com/frdeso/syscall-bench-it.git",
                        "revision": "master",
                        "testdef": "lava/testcases/failing-open-efault.yml"
                    },
                    {
                        "git-repo": "https://github.com/frdeso/syscall-bench-it.git",
                        "revision": "master",
                        "testdef": "lava/testcases/failing-open-enoent.yml"
                    }
                ],
                "timeout": 18000
            }
        },
         {
            "command": "submit_results",
            "parameters": {
                "server": "http://lava-master.internal.efficios.com/RPC2/",
                "stream": "/anonymous/benchmark-kernel/"
            }
        }
    ]
}
"""
# We use the kernel image and modules archive received as argument
deploy_action={"command": "deploy_kernel",
            "parameters": {
            		"overlays": [
            		    "scp://jenkins-lava@storage.internal.efficios.com"+modules ],
                "kernel":
                "scp://jenkins-lava@storage.internal.efficios.com"+kernel,
                "nfsrootfs": "scp://jenkins-lava@storage.internal.efficios.com/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz",
                "target_type": "ubuntu"
            }
        }

# We checkout the commit id for tools
setup_action = {
            "command": "lava_command_run",
            "parameters": {
                "commands": [
                	"git clone https://github.com/frdeso/syscall-bench-it.git bm",
                    "pip3 install vlttng",
                    "vlttng --jobs=16 --profile urcu-master \
                    --profile lttng-tools-master -o \
                    projects.lttng-tools.checkout="+tools_commit+ \
                     " /tmp/virtenv"
                ]
            }
        }
job_dict= json.loads(job)
job_dict['actions'].insert(0, deploy_action)
job_dict['actions'].insert(4, setup_action)

username = 'frdeso'
hostname = 'lava-master.internal.efficios.com'
server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (username, token, hostname))

print server.scheduler.submit_job(json.dumps(job_dict))
<|endoftext|>


<commit_before>import os
import re
import sys
import time
import json
import fuzzy
import urllib
import urllib2
import codecs
import base64

class leGenderary:

    def __init__(self, options):
        self.options = options
        self.firstDict = self.parseFirstDataSet(options['dict1'])
        self.secondDict = self.parseSecondDataSet(options['dict2'])

    def determineFirstName(self, nameArray):
        honorifics = ['mr', 'ms', 'md', 'mrs', 'dr', 'st', 'lcol', 'prince',
                      'mister', 'master', 'rev', 'revd', 'capt', 'lt',
                      'sri', 'smt', 'thiru', 'sir', 'maam', 'major', 'lord']

        length     = len(nameArray)
        cleanFirst = re.sub("[^A-Za-z]", "", nameArray[0]).lower()

        if length == 1:
            return nameArray[0]
        elif cleanFirst in honorifics:
            return self.determineFirstName(nameArray[1:])
        elif length >= 2:
            if len(cleanFirst) > 1:
                return nameArray[0]
            else:
                return ''
        else:
            return ' '.join(nameArray)

    def gPetersDotCom(self, name):
        url    = 'http://www.gpeters.com/names/baby-names.php?name=' + self._sanitizeName(name)
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        get    = opener.open(url).read()

        try:
            findGender  = self._cut("It's a ", '', get)
            findGender  = self.options['male'] if findGender.find('boy') == 0 else self.options['female']
            probability = self._cut("Based on popular usage", " times more common", get)
            probability = float(self._stripTags(probability).split()[-1])

            if probability < 1.20:
                gender = self.options['androgynous']
            else:
                gender = findGender
        except:
            gender = self.options['unknown']

        return gender

    def determineFromDictionary(self, firstName):
        firstName = self._sanitizeName(firstName)

        if firstName in self.firstDict:
            return self.firstDict[firstName]

        if firstName in self.secondDict:
            return self.secondDict[firstName]

        return self.options['unknown']

    def determineFromSoundex(self, firstName):
        hashTable = {}
        self.generateSoundexHash(self.firstDict, hashTable)
        self.generateSoundexHash(self.secondDict, hashTable)

        firstName = self._sanitizeName(firstName)
        nameHash  = fuzzy.Soundex(4)(firstName)

        if nameHash in hashTable:
            results = hashTable[nameHash]
            gender  = max(results, key=results.get)

            if results[gender] > 0:
                return gender

        return self.options['unknown']

    def parseFirstDataSet(self, fileName):
        names = {}
        f = codecs.open(fileName, 'r', encoding='iso8859-1')
        line = f.readline()

        while line:
            if line.startswith("#") or line.startswith("="):
                pass

            parts = filter(lambda p: p.strip() != "", line.split(" "))

            if "F" in parts[0]:
                name = parts[1].lower()
                gender = self.options['female']
            elif "M" in parts[0]:
                name = parts[1].lower()
                gender = self.options['male']
            else:
                name = parts[1].lower()
                gender = self.options['androgynous']

            if names.has_key(name):
                pass
            if "+" in name:
                for replacement in [ '', '-', ' ' ]:
                    name = name.replace('+', replacement).lower()
            else:
                names[name] = gender

            line = f.readline()

        f.close()

        return names

    def parseSecondDataSet(self, fileName):
        names = {}
        f = codecs.open(fileName, 'r', encoding='iso8859-1').read()

        for person in f.split("\n"):
            try:
                separate = person.split(',')
                name = separate[0].lower()

                if int(separate[1]) == 0:
                    gender = self.options['male']
                elif int(separate[1]) == 1:
                    gender = self.options['female']
                else:
                    gender = self.options['androgynous']

                names[name] = gender
            except:
                pass

        return names

    def generateSoundexHash(self, dictionary, table=None):
        soundexHash = {} if table is None else table

        for name, gender in dictionary.iteritems():
            name = self._sanitizeName(name)

            if len(name) > 1:
                soundhash = fuzzy.Soundex(4)(name)
                self._addToDict(soundhash, gender, soundexHash)

        return soundexHash

    def generateNysiisHash(self, dictionary, table=None):
        nysiisHash = {} if table is None else table

        for name, gender in dictionary.iteritems():
            name = self._sanitizeName(name)

            if len(name) > 1:
                nysiishash = fuzzy.nysiis(name)
                self._addToDict(nysiishash, gender, nysiisHash)

        return nysiisHash

    def generateMetaphoneHash(self, dictionary, table=None):
        metaphoneHash = {} if table is None else table

        for name, gender in dictionary.iteritems():
            name = self._sanitizeName(name)

            if len(name) > 1:
                metaphonehash = fuzzy.DMetaphone()(name)
                self._addToDict(metaphonehash, gender, metaphoneHash)

        return metaphoneHash

    def _addToDict(self, soundhash, gender, array):
        if type(soundhash) in [str, unicode]:
            soundhash = [soundhash]

        male = self.options['male']
        female = self.options['female']

        for i in soundhash:
            if i != None:
                if len(i) < 2:
                    break

                if gender == male:
                    if i in array:
                        array[i] = {str(male)   : 1 + array[i][str(male)],
                                    str(female) : array[i][str(female)]}
                    else:
                        array[i] = {str(male)   : 0,
                                    str(female) : 0}

                if gender == female:
                    if i in array:
                        array[i] = {str(male)   : array[i][str(male)],
                                    str(female) : 1 + array[i][str(female)]}
                    else:
                        array[i] = {str(male)   : 0,
                                    str(female) : 0}

    def _cut(self, start, end, data):
        rez = []
        one = data.split(start)

        for i in range(1, len(one)):
            two = one[i].split(end)
            rez.append(two[0])

        return rez[0] if len(rez) == 1 else rez

    def _stripTags(self, html):
        return re.sub("(<[^>]*?>|\n|\r|\t)", '', html)

    def _sanitizeName(self, name):
        name = name.lower()
        name = re.sub("[^A-Za-z]", "", name)
        return name

if __name__ == '__main__':

    options = { 'male'          : 'male',
                'female'        : 'female',
                'androgynous'   : 'androgynous',
                'unknown'       : 'unknown',
                'maleConfirm'   : 'male needs confirmation',
                'femaleConfirm' : 'female needs confirmation',
                'dict1'         : 'dict1.txt',
                'dict2'         : 'dict2.txt',
               }

    gender = leGenderary(options)
    fullName = "Dr. Richard P. Feynman"

    firstName  = gender.determineFirstName(fullName.split())
    gPeters    = gender.gPetersDotCom(firstName)
    dictionary = gender.determineFromDictionary(firstName)
    soundexH   = gender.generateSoundexHash(gender.secondDict)
    nysiishH   = gender.generateNysiisHash(gender.secondDict)
    metaphoneH = gender.generateMetaphoneHash(gender.secondDict)
    soundex    = gender.determineFromSoundex('Rikard')

    print soundex

<commit_msg>Added Nysiis method<commit_after>import os
import re
import sys
import time
import json
import fuzzy
import urllib
import urllib2
import codecs
import base64

class leGenderary:

    def __init__(self, options):
        self.options = options
        self.firstDict = self.parseFirstDataSet(options['dict1'])
        self.secondDict = self.parseSecondDataSet(options['dict2'])

    def determineFirstName(self, nameArray):
        honorifics = ['mr', 'ms', 'md', 'mrs', 'dr', 'st', 'lcol', 'prince',
                      'mister', 'master', 'rev', 'revd', 'capt', 'lt',
                      'sri', 'smt', 'thiru', 'sir', 'maam', 'major', 'lord']

        length     = len(nameArray)
        cleanFirst = re.sub("[^A-Za-z]", "", nameArray[0]).lower()

        if length == 1:
            return nameArray[0]
        elif cleanFirst in honorifics:
            return self.determineFirstName(nameArray[1:])
        elif length >= 2:
            if len(cleanFirst) > 1:
                return nameArray[0]
            else:
                return ''
        else:
            return ' '.join(nameArray)

    def gPetersDotCom(self, name):
        url    = 'http://www.gpeters.com/names/baby-names.php?name=' + self._sanitizeName(name)
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        get    = opener.open(url).read()

        try:
            findGender  = self._cut("It's a ", '', get)
            findGender  = self.options['male'] if findGender.find('boy') == 0 else self.options['female']
            probability = self._cut("Based on popular usage", " times more common", get)
            probability = float(self._stripTags(probability).split()[-1])

            if probability < 1.20:
                gender = self.options['androgynous']
            else:
                gender = findGender
        except:
            gender = self.options['unknown']

        return gender

    def determineFromDictionary(self, firstName):
        firstName = self._sanitizeName(firstName)

        if firstName in self.firstDict:
            return self.firstDict[firstName]

        if firstName in self.secondDict:
            return self.secondDict[firstName]

        return self.options['unknown']

    def determineFromSoundex(self, firstName):
        hashTable = {}
        self.generateSoundexHash(self.firstDict, hashTable)
        self.generateSoundexHash(self.secondDict, hashTable)

        firstName = self._sanitizeName(firstName)
        nameHash  = fuzzy.Soundex(4)(firstName)

        if nameHash in hashTable:
            results = hashTable[nameHash]
            gender  = max(results, key=results.get)

            if results[gender] > 0:
                return gender

        return self.options['unknown']

    def determineFromNysiis(self, firstName):
        hashTable = {}
        self.generateNysiisHash(self.firstDict, hashTable)
        self.generateNysiisHash(self.secondDict, hashTable)

        firstName = self._sanitizeName(firstName)
        nameHash  = fuzzy.nysiis(firstName)

        if nameHash in hashTable:
            results = hashTable[nameHash]
            gender  = max(results, key=results.get)

            if results[gender] > 0:
                return gender

        return self.options['unknown']

    def parseFirstDataSet(self, fileName):
        names = {}
        f = codecs.open(fileName, 'r', encoding='iso8859-1')
        line = f.readline()

        while line:
            if line.startswith("#") or line.startswith("="):
                pass

            parts = filter(lambda p: p.strip() != "", line.split(" "))

            if "F" in parts[0]:
                name = parts[1].lower()
                gender = self.options['female']
            elif "M" in parts[0]:
                name = parts[1].lower()
                gender = self.options['male']
            else:
                name = parts[1].lower()
                gender = self.options['androgynous']

            if names.has_key(name):
                pass
            if "+" in name:
                for replacement in [ '', '-', ' ' ]:
                    name = name.replace('+', replacement).lower()
            else:
                names[name] = gender

            line = f.readline()

        f.close()

        return names

    def parseSecondDataSet(self, fileName):
        names = {}
        f = codecs.open(fileName, 'r', encoding='iso8859-1').read()

        for person in f.split("\n"):
            try:
                separate = person.split(',')
                name = separate[0].lower()

                if int(separate[1]) == 0:
                    gender = self.options['male']
                elif int(separate[1]) == 1:
                    gender = self.options['female']
                else:
                    gender = self.options['androgynous']

                names[name] = gender
            except:
                pass

        return names

    def generateSoundexHash(self, dictionary, table=None):
        soundexHash = {} if table is None else table

        for name, gender in dictionary.iteritems():
            name = self._sanitizeName(name)

            if len(name) > 1:
                soundhash = fuzzy.Soundex(4)(name)
                self._addToDict(soundhash, gender, soundexHash)

        return soundexHash

    def generateNysiisHash(self, dictionary, table=None):
        nysiisHash = {} if table is None else table

        for name, gender in dictionary.iteritems():
            name = self._sanitizeName(name)

            if len(name) > 1:
                nysiishash = fuzzy.nysiis(name)
                self._addToDict(nysiishash, gender, nysiisHash)

        return nysiisHash

    def generateMetaphoneHash(self, dictionary, table=None):
        metaphoneHash = {} if table is None else table

        for name, gender in dictionary.iteritems():
            name = self._sanitizeName(name)

            if len(name) > 1:
                metaphonehash = fuzzy.DMetaphone()(name)
                self._addToDict(metaphonehash, gender, metaphoneHash)

        return metaphoneHash

    def _addToDict(self, soundhash, gender, array):
        if type(soundhash) in [str, unicode]:
            soundhash = [soundhash]

        male = self.options['male']
        female = self.options['female']

        for i in soundhash:
            if i != None:
                if len(i) < 2:
                    break

                if gender == male:
                    if i in array:
                        array[i] = {str(male)   : 1 + array[i][str(male)],
                                    str(female) : array[i][str(female)]}
                    else:
                        array[i] = {str(male)   : 0,
                                    str(female) : 0}

                if gender == female:
                    if i in array:
                        array[i] = {str(male)   : array[i][str(male)],
                                    str(female) : 1 + array[i][str(female)]}
                    else:
                        array[i] = {str(male)   : 0,
                                    str(female) : 0}

    def _cut(self, start, end, data):
        rez = []
        one = data.split(start)

        for i in range(1, len(one)):
            two = one[i].split(end)
            rez.append(two[0])

        return rez[0] if len(rez) == 1 else rez

    def _stripTags(self, html):
        return re.sub("(<[^>]*?>|\n|\r|\t)", '', html)

    def _sanitizeName(self, name):
        name = name.lower()
        name = re.sub("[^A-Za-z]", "", name)
        return name

if __name__ == '__main__':

    options = { 'male'          : 'male',
                'female'        : 'female',
                'androgynous'   : 'androgynous',
                'unknown'       : 'unknown',
                'maleConfirm'   : 'male needs confirmation',
                'femaleConfirm' : 'female needs confirmation',
                'dict1'         : 'dict1.txt',
                'dict2'         : 'dict2.txt',
               }

    gender = leGenderary(options)
    fullName = "Dr. Richard P. Feynman"

    firstName  = gender.determineFirstName(fullName.split())
    gPeters    = gender.gPetersDotCom(firstName)
    dictionary = gender.determineFromDictionary(firstName)
    soundexH   = gender.generateSoundexHash(gender.secondDict)
    nysiishH   = gender.generateNysiisHash(gender.secondDict)
    metaphoneH = gender.generateMetaphoneHash(gender.secondDict)
    soundex    = gender.determineFromSoundex('Rikard')
    nysiis     = gender.determineFromNysiis('Rikard')

    print nysiis

<|endoftext|>


<commit_before>#!/opt/local/bin/python2.7

import os, sys, pygame
pygame.init()

black = 0, 0, 0

screen = pygame.display.set_mode((0,0), pygame.FULLSCREEN)
pygame.mouse.set_visible(False)
info = pygame.display.Info()
width = info.current_w
height = info.current_h

class NoneSound:
    def play(self): pass

class Thing:
    """A thing that contains an image and a sound"""
    def __init__(self, category, name):
        self.name = name
        self.category = category
        self.load()

    def load(self):
        self.loadsound()
        self.loadimage()

    def loadimage(self):
        imagepath = os.path.join("media", self.category, "photos", self.name + ".jpg")
        try:
            self.image = pygame.image.load(imagepath)
        except pygame.error, message:
            raise SystemExit, message
        self.image = self.image.convert()
        self.pos = ( (width - self.image.get_width())/2, (height - self.image.get_height())/2 )

    def loadsound(self):
        if not pygame.mixer:
            self.sound = NoneSound()
        else:
            soundpath = os.path.join("media", self.category, "sounds", self.name     + ".ogg")
            self.sound = pygame.mixer.Sound(soundpath)

    def show(self):
        screen.fill(black)
        screen.blit(self.image, self.pos)
        pygame.display.flip()
        self.sound.play()

rooster = Thing("animals", "rooster")
sheep = Thing("animals", "sheep")

rooster.show()

while 1:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                sys.exit()
            else:
                sheep.show()
<commit_msg>scale images, exit on C-q, count keypresses<commit_after>#!/opt/local/bin/python2.7

import os, sys, pygame
from pygame.locals import *
pygame.init()

black = 0, 0, 0

screen = pygame.display.set_mode((0,0), pygame.FULLSCREEN)
pygame.mouse.set_visible(False)
info = pygame.display.Info()
width = info.current_w
height = info.current_h

class NoneSound:
    def play(self): pass

class Thing:
    """A thing that contains an image and a sound"""
    def __init__(self, category, name):
        self.name = name
        self.category = category
        self.load()

    def load(self):
        self.loadsound()
        self.loadimage()

    def loadimage(self):
        imagepath = os.path.join("media", self.category, "photos", self.name + ".jpg")
        try:
            self.image = pygame.image.load(imagepath)
        except pygame.error, message:
            raise SystemExit, message
        self.image = self.image.convert()
        self.image = pygame.transform.smoothscale(self.image, (width, height))

    def loadsound(self):
        if not pygame.mixer:
            self.sound = NoneSound()
        else:
            soundpath = os.path.join("media", self.category, "sounds", self.name     + ".ogg")
            self.sound = pygame.mixer.Sound(soundpath)

    def show(self):
        screen.fill(black)
        screen.blit(self.image, (0,0))
        pygame.display.flip()
        self.sound.play()

rooster = Thing("animals", "rooster")
sheep = Thing("animals", "sheep")

rooster.show()

keypresses = 0

while 1:
    if keypresses > 10:
        keypresses = 0
        sheep.show()
    for event in pygame.event.get():
        if event.type == QUIT:
            sys.exit()
        elif event.type == KEYDOWN:
            if event.mod & KMOD_CTRL and event.key == K_q:
                sys.exit()
            else:
                keypresses = keypresses + 1
<|endoftext|>


<commit_before>from collections import Iterable
from copy import deepcopy
from functools import reduce
from importlib import import_module

from django.conf.urls import url
from django.http import HttpResponse
from jsonschema import RefResolver

from lepo.excs import MissingHandler
from lepo.path import Path
from lepo.utils import maybe_resolve, snake_case

def root_view(request):
    return HttpResponse('API root')

class Router:
    path_class = Path

    def __init__(self, api):
        self.api = deepcopy(api)
        self.api.pop('host', None)
        self.handlers = {}
        self.resolver = RefResolver('', self.api)

    @classmethod
    def from_file(cls, filename):
        with open(filename) as infp:
            if filename.endswith('.yaml') or filename.endswith('.yml'):
                import yaml
                data = yaml.safe_load(infp)
            else:
                import json
                data = json.load(infp)
        return cls(data)

    def get_path(self, path):
        """
        Construct a Path object from a path string.

        The Path string must be declared in the API.

        :type path: str
        :rtype: lepo.path.Path
        """
        mapping = maybe_resolve(self.api['paths'][path], self.resolve_reference)
        return self.path_class(router=self, path=path, mapping=mapping)

    def get_paths(self):
        for path in self.api['paths']:
            yield self.get_path(path)

    def get_urls(
        self,
        root_view_name=None,
        optional_trailing_slash=False,
        decorate=(),
        name_template='{name}',
    ):
        """
        Get the router's URLs, ready to be installed in `urlpatterns` (directly or via `include`).

        :param root_view_name: The optional url name for an API root view.
                               This may be useful for projects that do not explicitly know where the
                               router is mounted; those projects can then use `reverse('api:root')`,
                               for instance, if they need to construct URLs based on the API's root URL.
        :type root_view_name: str|None

        :param optional_trailing_slash: Whether to fix up the regexen for the router to make any trailing
                                        slashes optional.
        :type optional_trailing_slash: bool

        :param decorate: A function to decorate view functions with, or an iterable of such decorators.
                         Use `(lepo.decorators.csrf_exempt,)` to mark all API views as CSRF exempt.
        :type decorate: function|Iterable[function]

        :param name_template: A `.format()` template for view naming.
        :type name_template: str

        :return: List of URL tuples.
        :rtype: list[tuple]
        """
        if isinstance(decorate, Iterable):
            decorators = decorate

            def decorate(view):
                return reduce(lambda view, decorator: decorator(view), decorators, view)

        urls = []
        for path in self.get_paths():
            regex = path.regex
            if optional_trailing_slash:
                regex = regex.rstrip('$')
                if not regex.endswith('/'):
                    regex += '/'
                regex += '?$'
            view = decorate(path.view_class.as_view())
            urls.append(url(regex, view, name=name_template.format(name=path.name)))

        if root_view_name:
            urls.append(url(r'^$', root_view, name=name_template.format(name=root_view_name)))
        return urls

    def get_handler(self, operation_id):
        handler = (
            self.handlers.get(operation_id)
            or self.handlers.get(snake_case(operation_id))
        )
        if handler:
            return handler
        raise MissingHandler(
            'Missing handler for operation %s (tried %s too)' % (operation_id, snake_case(operation_id))
        )

    def add_handlers(self, namespace):
        if isinstance(namespace, str):
            namespace = import_module(namespace)
        for name, value in vars(namespace).items():
            if name.startswith('_'):
                continue
            try:
                if callable(value):
                    self.handlers[name] = value
            except:
                pass

    def resolve_reference(self, ref):
        url, resolved = self.resolver.resolve(ref)
        return resolved
<commit_msg>Improve handler discovery:<commit_after>from collections import Iterable
from copy import deepcopy
from functools import reduce
from importlib import import_module
from inspect import isfunction, ismethod

from django.conf.urls import url
from django.http import HttpResponse
from jsonschema import RefResolver

from lepo.excs import MissingHandler
from lepo.path import Path
from lepo.utils import maybe_resolve, snake_case

def root_view(request):
    return HttpResponse('API root')

class Router:
    path_class = Path

    def __init__(self, api):
        self.api = deepcopy(api)
        self.api.pop('host', None)
        self.handlers = {}
        self.resolver = RefResolver('', self.api)

    @classmethod
    def from_file(cls, filename):
        with open(filename) as infp:
            if filename.endswith('.yaml') or filename.endswith('.yml'):
                import yaml
                data = yaml.safe_load(infp)
            else:
                import json
                data = json.load(infp)
        return cls(data)

    def get_path(self, path):
        """
        Construct a Path object from a path string.

        The Path string must be declared in the API.

        :type path: str
        :rtype: lepo.path.Path
        """
        mapping = maybe_resolve(self.api['paths'][path], self.resolve_reference)
        return self.path_class(router=self, path=path, mapping=mapping)

    def get_paths(self):
        for path in self.api['paths']:
            yield self.get_path(path)

    def get_urls(
        self,
        root_view_name=None,
        optional_trailing_slash=False,
        decorate=(),
        name_template='{name}',
    ):
        """
        Get the router's URLs, ready to be installed in `urlpatterns` (directly or via `include`).

        :param root_view_name: The optional url name for an API root view.
                               This may be useful for projects that do not explicitly know where the
                               router is mounted; those projects can then use `reverse('api:root')`,
                               for instance, if they need to construct URLs based on the API's root URL.
        :type root_view_name: str|None

        :param optional_trailing_slash: Whether to fix up the regexen for the router to make any trailing
                                        slashes optional.
        :type optional_trailing_slash: bool

        :param decorate: A function to decorate view functions with, or an iterable of such decorators.
                         Use `(lepo.decorators.csrf_exempt,)` to mark all API views as CSRF exempt.
        :type decorate: function|Iterable[function]

        :param name_template: A `.format()` template for view naming.
        :type name_template: str

        :return: List of URL tuples.
        :rtype: list[tuple]
        """
        if isinstance(decorate, Iterable):
            decorators = decorate

            def decorate(view):
                return reduce(lambda view, decorator: decorator(view), decorators, view)

        urls = []
        for path in self.get_paths():
            regex = path.regex
            if optional_trailing_slash:
                regex = regex.rstrip('$')
                if not regex.endswith('/'):
                    regex += '/'
                regex += '?$'
            view = decorate(path.view_class.as_view())
            urls.append(url(regex, view, name=name_template.format(name=path.name)))

        if root_view_name:
            urls.append(url(r'^$', root_view, name=name_template.format(name=root_view_name)))
        return urls

    def get_handler(self, operation_id):
        handler = (
            self.handlers.get(operation_id)
            or self.handlers.get(snake_case(operation_id))
        )
        if handler:
            return handler
        raise MissingHandler(
            'Missing handler for operation %s (tried %s too)' % (operation_id, snake_case(operation_id))
        )

    def add_handlers(self, namespace):
        """
        Add handler functions from the given `namespace`, for instance a module.

        The namespace may be a string, in which case it is expected to be a name of a module.
        It may also be a dictionary mapping names to functions.

        Only non-underscore-prefixed functions and methods are imported.

        :param namespace: Namespace object.
        :type namespace: str|module|dict[str, function]
        """
        if isinstance(namespace, str):
            namespace = import_module(namespace)

        if isinstance(namespace, dict):
            namespace = namespace.items()
        else:
            namespace = vars(namespace).items()

        for name, value in namespace:
            if name.startswith('_'):
                continue
            if isfunction(value) or ismethod(value):
                self.handlers[name] = value

    def resolve_reference(self, ref):
        url, resolved = self.resolver.resolve(ref)
        return resolved
<|endoftext|>


<commit_before>import tensorflow as tf
from tensorflow.contrib.framework import arg_scope

from layers import *

class Model(object):
  def __init__(self, config,
               inputs, labels, enc_seq_length, dec_seq_length,
               reuse=False, is_critic=False):
    self.task = config.task
    self.debug = config.debug
    self.config = config

    self.input_dim = config.input_dim
    self.hidden_dim = config.hidden_dim
    self.num_layers = config.num_layers

    self.max_enc_length = config.max_enc_length
    self.max_dec_length = config.max_dec_length
    self.num_glimpse = config.num_glimpse

    self.init_min_val = config.init_min_val
    self.init_max_val = config.init_max_val
    self.initializer = \
        tf.random_uniform_initializer(self.init_min_val, self.init_max_val)

    self.use_terminal_symbol = config.use_terminal_symbol

    self.lr_start = config.lr_start
    self.lr_decay_step = config.lr_decay_step
    self.lr_decay_rate = config.lr_decay_rate
    self.max_grad_norm = config.max_grad_norm

    self.layer_dict = {}

    ##############
    # inputs
    ##############

    self.is_training = tf.placeholder_with_default(
        tf.constant(False, dtype=tf.bool),
        shape=(), name='is_training'
    )

    self.enc_inputs, self.dec_targets, self.enc_seq_length, self.dec_seq_length = \
        tf.contrib.layers.utils.smart_cond(
            self.is_training,
            lambda: (inputs['train'], labels['train'], enc_seq_length['train'], dec_seq_length['train']),
            lambda: (inputs['test'], labels['test'], enc_seq_length['test'], dec_seq_length['test'])
        )

    if self.use_terminal_symbol:
      self.dec_seq_length += 1 # terminal symbol
    self.dec_input_mask = tf.ones(self.dec_seq_length, dtype=tf.int32)

    self._build_model()
    if is_critic:
      self._build_critic_model()

    #if not reuse:
    #  self._build_optim()
    #  self._build_summary()

  def _build_summary(self):
    tf.summary.scalar("learning_rate", self.lr)

  def _build_model(self):
    tf.logging.info("Create a model..")
    self.global_step = tf.Variable(0, trainable=False)

    input_embed = tf.get_variable(
        "input_embed", [1, self.input_dim, self.hidden_dim],
        initializer=self.initializer)

    with tf.variable_scope("encoder"):
      self.embeded_enc_inputs = tf.nn.conv1d(
          self.enc_inputs, input_embed, 1, "VALID")

    batch_size = tf.shape(self.enc_inputs)[0]
    with tf.variable_scope("encoder"):
      self.enc_cell = LSTMCell(
          self.hidden_dim,
          initializer=self.initializer)

      if self.num_layers > 1:
        cells = [self.enc_cell] * self.num_layers
        self.enc_cell = MultiRNNCell(cells)
      self.enc_init_state = trainable_initial_state(
          batch_size, self.enc_cell.state_size)

      # self.encoder_outputs : [None, max_time, output_size]
      self.enc_outputs, self.enc_final_states = tf.nn.dynamic_rnn(
          self.enc_cell, self.embeded_enc_inputs,
          self.enc_seq_length, self.enc_init_state)

      if self.use_terminal_symbol:
        # 0 index indicates terminal
        first_decoder_input = tf.expand_dims(trainable_initial_state(
            batch_size, self.hidden_dim, name="first_decoder_input"), 1)
        self.enc_outputs = tf.concat_v2(
            [first_decoder_input, self.enc_outputs], axis=1)

    with tf.variable_scope("dencoder"):
      if self.use_terminal_symbol:
        tiled_zero_idxs = tf.tile(tf.zeros(
            [1, 1], dtype=tf.int32), [batch_size, 1], name="tiled_zero_idxs")
        self.dec_targets = tf.concat_v2([self.dec_targets, tiled_zero_idxs], axis=1)

      self.idx_pairs = index_matrix_to_pairs(self.dec_targets)
      self.embeded_dec_inputs = tf.gather_nd(self.enc_outputs, self.idx_pairs)

      self.dec_cell = LSTMCell(
          self.hidden_dim,
          initializer=self.initializer)

      if self.num_layers > 1:
        cells = [self.dec_cell] * self.num_layers
        self.dec_cell = MultiRNNCell(cells)

      self.dec_output_logits, self.dec_states, _ = decoder_rnn(
          self.dec_cell, self.embeded_dec_inputs,
          self.enc_outputs, self.enc_final_states,
          self.dec_seq_length, self.hidden_dim, self.num_glimpse,
          self.max_dec_length, batch_size, is_train=True,
          initializer=self.initializer)

    with tf.variable_scope("dencoder", reuse=True):
      self.dec_outputs, _, self.predictions = decoder_rnn(
          self.dec_cell, first_decoder_input,
          self.enc_outputs, self.enc_final_states,
          self.dec_seq_length, self.hidden_dim, self.num_glimpse,
          self.max_dec_length, batch_size, is_train=False,
          initializer=self.initializer)

  def _build_critic_model(self):
    pass

  def _build_optim(self):
    losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
        labels=self.dec_targets, logits=self.dec_output_logits)

    mask = self.dec_input_mask
    batch_loss = tf.div(tf.reduce_sum(tf.multiply(losses, mask)),
                        tf.reduce_sum(mask), name="batch_loss")

    tf.losses.add_loss(batch_loss)
    total_loss = tf.losses.get_total_loss()

    tf.summary.scalar("losses/batch_loss", batch_loss)
    tf.summary.scalar("losses/total_loss", total_loss)

    # TODO: length masking
    #mask = tf.sign(tf.to_float(targets_flat))
    #masked_losses = mask * self.loss

    self.lr = tf.train.exponential_decay(
        self.lr_start, self.global_step, self.lr_decay_step,
        self.lr_decay_rate, staircase=True, name="learning_rate")

    optimizer = tf.train.AdamOptimizer(self.lr)

    if self.max_grad_norm != None:
      grads_and_vars = optimizer.compute_gradients(self.loss)
      for idx, (grad, var) in enumerate(grads_and_vars):
        if grad is not None:
          grads_and_vars[idx] = (tf.clip_by_norm(grad, self.max_grad_norm), var)
      self.optim = optimizer.apply_gradients(grads_and_vars)
    else:
      self.optim = optimizer.minimize(self.loss)
<commit_msg>now loss masking in progress<commit_after>import tensorflow as tf
from tensorflow.contrib.framework import arg_scope

from layers import *

class Model(object):
  def __init__(self, config,
               inputs, labels, enc_seq_length, dec_seq_length,
               reuse=False, is_critic=False):
    self.task = config.task
    self.debug = config.debug
    self.config = config

    self.input_dim = config.input_dim
    self.hidden_dim = config.hidden_dim
    self.num_layers = config.num_layers

    self.max_enc_length = config.max_enc_length
    self.max_dec_length = config.max_dec_length
    self.num_glimpse = config.num_glimpse

    self.init_min_val = config.init_min_val
    self.init_max_val = config.init_max_val
    self.initializer = \
        tf.random_uniform_initializer(self.init_min_val, self.init_max_val)

    self.use_terminal_symbol = config.use_terminal_symbol

    self.lr_start = config.lr_start
    self.lr_decay_step = config.lr_decay_step
    self.lr_decay_rate = config.lr_decay_rate
    self.max_grad_norm = config.max_grad_norm

    self.layer_dict = {}

    ##############
    # inputs
    ##############

    self.is_training = tf.placeholder_with_default(
        tf.constant(False, dtype=tf.bool),
        shape=(), name='is_training'
    )

    self.enc_inputs, self.dec_targets, self.enc_seq_length, self.dec_seq_length = \
        tf.contrib.layers.utils.smart_cond(
            self.is_training,
            lambda: (inputs['train'], labels['train'], enc_seq_length['train'], dec_seq_length['train']),
            lambda: (inputs['test'], labels['test'], enc_seq_length['test'], dec_seq_length['test'])
        )

    if self.use_terminal_symbol:
      self.dec_seq_length += 1 # terminal symbol

    self._build_model()
    if is_critic:
      self._build_critic_model()

    if not reuse:
      self._build_optim()

  def _build_model(self):
    tf.logging.info("Create a model..")
    self.global_step = tf.Variable(0, trainable=False)

    input_embed = tf.get_variable(
        "input_embed", [1, self.input_dim, self.hidden_dim],
        initializer=self.initializer)

    with tf.variable_scope("encoder"):
      self.embeded_enc_inputs = tf.nn.conv1d(
          self.enc_inputs, input_embed, 1, "VALID")

    batch_size = tf.shape(self.enc_inputs)[0]
    with tf.variable_scope("encoder"):
      self.enc_cell = LSTMCell(
          self.hidden_dim,
          initializer=self.initializer)

      if self.num_layers > 1:
        cells = [self.enc_cell] * self.num_layers
        self.enc_cell = MultiRNNCell(cells)
      self.enc_init_state = trainable_initial_state(
          batch_size, self.enc_cell.state_size)

      # self.encoder_outputs : [None, max_time, output_size]
      self.enc_outputs, self.enc_final_states = tf.nn.dynamic_rnn(
          self.enc_cell, self.embeded_enc_inputs,
          self.enc_seq_length, self.enc_init_state)

      if self.use_terminal_symbol:
        # 0 index indicates terminal
        first_decoder_input = tf.expand_dims(trainable_initial_state(
            batch_size, self.hidden_dim, name="first_decoder_input"), 1)
        self.enc_outputs = tf.concat_v2(
            [first_decoder_input, self.enc_outputs], axis=1)

    with tf.variable_scope("dencoder"):
      if self.use_terminal_symbol:
        tiled_zero_idxs = tf.tile(tf.zeros(
            [1, 1], dtype=tf.int32), [batch_size, 1], name="tiled_zero_idxs")
        self.dec_targets = tf.concat_v2([self.dec_targets, tiled_zero_idxs], axis=1)

      self.idx_pairs = index_matrix_to_pairs(self.dec_targets)
      self.embeded_dec_inputs = tf.gather_nd(self.enc_outputs, self.idx_pairs)

      self.dec_cell = LSTMCell(
          self.hidden_dim,
          initializer=self.initializer)

      if self.num_layers > 1:
        cells = [self.dec_cell] * self.num_layers
        self.dec_cell = MultiRNNCell(cells)

      self.dec_output_logits, self.dec_states, _ = decoder_rnn(
          self.dec_cell, self.embeded_dec_inputs,
          self.enc_outputs, self.enc_final_states,
          self.dec_seq_length, self.hidden_dim, self.num_glimpse,
          self.max_dec_length, batch_size, is_train=True,
          initializer=self.initializer)

    with tf.variable_scope("dencoder", reuse=True):
      self.dec_outputs, _, self.predictions = decoder_rnn(
          self.dec_cell, first_decoder_input,
          self.enc_outputs, self.enc_final_states,
          self.dec_seq_length, self.hidden_dim, self.num_glimpse,
          self.max_dec_length, batch_size, is_train=False,
          initializer=self.initializer)

  def _build_critic_model(self):
    pass

  def _build_optim(self):
    losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
        labels=self.dec_targets, logits=self.dec_output_logits)

    def apply_mask(op):
      length = op[0]
      lossses = op[1:]
      return tf.multiply(losses, tf.ones(length, dtype=tf.float32)

    import ipdb; ipdb.set_trace()
    tf.expand_dims(self.dec_seq_length, 1)
    self.mask = tf.map_fn(
        apply_mask, tf.expand_dims(self.dec_seq_length, 1), dtype=tf.float32)
    batch_loss = tf.div(tf.reduce_sum(tf.multiply(losses, self.mask)),
                        tf.reduce_sum(self.mask), name="batch_loss")

    tf.losses.add_loss(batch_loss)
    total_loss = tf.losses.get_total_loss()

    tf.summary.scalar("losses/batch_loss", batch_loss)
    tf.summary.scalar("losses/total_loss", total_loss)

    self.total_loss = total_loss
    self.target_cross_entropy_losses = losses

    self.lr = tf.train.exponential_decay(
        self.lr_start, self.global_step, self.lr_decay_step,
        self.lr_decay_rate, staircase=True, name="learning_rate")

    optimizer = tf.train.AdamOptimizer(self.lr)

    if self.max_grad_norm != None:
      grads_and_vars = optimizer.compute_gradients(self.total_loss)
      for idx, (grad, var) in enumerate(grads_and_vars):
        if grad is not None:
          grads_and_vars[idx] = (tf.clip_by_norm(grad, self.max_grad_norm), var)
      self.optim = optimizer.apply_gradients(grads_and_vars)
    else:
      self.optim = optimizer.minimize(self.total_loss)
<|endoftext|>


<commit_before># coding: UTF-8
# Wikidata to CSV
import rdflib

g = rdflib.ConjunctiveGraph(store="SPARQLStore")
g.store.endpoint = "https://query.wikidata.org/bigdata/namespace/wdq/sparql"
info = g.query('''
SELECT ?code ?name ?site WHERE {
 ?s wdt:P429 ?code ; # 全国地方公共団体コード
    rdfs:label ?name .
  OPTIONAL {
    ?s p:P856 ?stmt .
    ?stmt ps:P856 ?site . # 公式サイト
  }
 FILTER ( lang(?name)="ja" )
 FILTER NOT EXISTS { ?s wdt:P31 wd:Q18663566 } # 分類 日本の廃止市町村
 FILTER NOT EXISTS { ?s wdt:P31 wd:Q850450 } # 分類 支庁
 FILTER NOT EXISTS { ?stmt pq:P642 ?x } # ～についての
} ORDER BY ?code ?site
''')
with open("docs/wd.csv","w") as f:
	f.write(info.serialize(format="csv").decode("UTF-8"))
<commit_msg>update the sparql query filter<commit_after># coding: UTF-8
# Wikidata to CSV
import rdflib

g = rdflib.ConjunctiveGraph(store="SPARQLStore")
g.store.endpoint = "https://query.wikidata.org/bigdata/namespace/wdq/sparql"
info = g.query('''
SELECT ?code ?name ?site WHERE {
 ?s wdt:P429 ?code ; # 全国地方公共団体コード
    rdfs:label ?name .
  OPTIONAL {
    ?s p:P856 ?stmt .
    ?stmt ps:P856 ?site . # 公式サイト
    FILTER NOT EXISTS { ?stmt pq:P642 ?x } # ～についての
    FILTER NOT EXISTS { ?stmt pq:P582 ?y } # 終了日
  }
 FILTER ( lang(?name)="ja" )
 FILTER NOT EXISTS { ?s wdt:P31 wd:Q18663566 } # 分類 日本の廃止市町村
 FILTER NOT EXISTS { ?s wdt:P31 wd:Q850450 } # 分類 支庁
} ORDER BY ?code ?site
''')
with open("docs/wd.csv","w") as f:
	f.write(info.serialize(format="csv").decode("UTF-8"))

# How to update the official website:
# - If the item has a new site, add an additional statement with preferred rank.
#   How to set preferred rank? See Help:Ranking#How_to_apply_ranks.
# - If a website is no longer valid, you could also:
#     qualify the URL with 終了日 (P582). If you don't know the exact date, use the year or "unknown" as date
#     add the qualifier アーカイブURL (P1065) to link to the former website at web.archive.org
# - Do not delete or replace the former URL.
<|endoftext|>


<commit_before>from telebot import types as types
import bson

# adds Dictionaryable behavior and marks models which can be stored in db
class Model(types.Dictionaryable):
    def to_dic(self):
        d = {}
        for k, v in vars(self).items():
            if isinstance(v, Model):
                d[k] = v.to_dic()
            elif not str(k).startswith('_'):
                if str(k) == 'id':
                    k = '_id'
                d[k] = v
        return d

# Represents User and adds extra fields to telepot's class
class User(Model, types.User):
    def __init__(self, *args, **kwargs):
        if not args:
            args = (
                kwargs.get('_id') or kwargs.get('id'),
                kwargs['first_name'],
                kwargs.get('last_name'),
                kwargs.get('username'),
            )
        self.blocked = kwargs.get('blocked')
        super().__init__(*args)

    def update(self, data):
        assert self.id == str(data.id)
        self.first_name = data.first_name
        self.last_name = data.last_name
        self.username = data.username

# Represents message, add ObjectId
class Message(Model, types.Message):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.id = bson.ObjectId()

# Just adds Dictionaryable to chat
class Chat(Model, types.Chat):
    pass

# Replaces classes in telepot.types
def replace_classes():
    types.User = User
    types.Message = Message
    types.Chat = Chat
<commit_msg>Special message serialization<commit_after>from telebot import types as types
import bson
import base64

import config

def to_python(value):
    try:
        return bson.ObjectId(base64.urlsafe_b64decode(value))
    except (bson.errors.InvalidId, ValueError, TypeError):
        raise ValueError

def to_url(value):
    return base64.urlsafe_b64encode(value.binary)

def short_id(value):
    oid = bson.ObjectId(str(value))
    time = int(oid.generation_time.timestamp()) * 1000
    print(time)
    counter = int(str(oid)[-6:], 16)
    counter = int(str(counter)[-3:], 10)
    print(counter)
    time += counter
    print(time)
    s = base64.urlsafe_b64encode(time.to_bytes(6, 'big'))
    s = s[1:]
    s = s[::-1]
    return s

# adds Dictionaryable behavior and marks models which can be stored in db
class Model(types.Dictionaryable):
    def to_dic(self):
        d = {}
        for k, v in vars(self).items():
            if isinstance(v, Model):
                d[k] = v.to_dic()
            elif not str(k).startswith('_'):
                if str(k) == 'id':
                    k = '_id'
                d[k] = v
        return d

# Represents User and adds extra fields to telepot's class
class User(Model, types.User):
    def __init__(self, *args, **kwargs):
        if not args:
            args = (
                kwargs.get('_id') or kwargs.get('id'),
                kwargs['first_name'],
                kwargs.get('last_name'),
                kwargs.get('username'),
            )
        self.blocked = kwargs.get('blocked')
        super().__init__(*args)

    def update(self, data):
        assert self.id == str(data.id)
        self.first_name = data.first_name
        self.last_name = data.last_name
        self.username = data.username

# Represents message, add ObjectId
class Message(Model, types.Message):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.id = bson.ObjectId()

    def to_dic(self):
        d = dict()
        d['_id'] = self.id
        d['tg_id'] = self.message_id
        d['short_id'] = short_id(self.id)
        d['from_user'] = self.from_user.id
        if self.from_user.id == config.my_id:
            d['with'] = self.reply_to_message.from_user.id
        else:
            d['with'] = self.from_user.id

        if self.text:
            d['text'] = self.text
        else:
            d['text'] = 'Non text message: /m' + short_id(self.id)
        return d

# Just adds Dictionaryable to chat
class Chat(Model, types.Chat):
    pass

# Replaces classes in telepot.types
def replace_classes():
    types.User = User
    types.Message = Message
    types.Chat = Chat
<|endoftext|>


<commit_before>import tensorflow as tf
import tensorflow.contrib.slim as slim
import utils
import shutil
from tqdm import tqdm
import os

"""
An implementation of the neural network used for
super-resolution of images as described in:

`Enhanced Deep Residual Networks for Single Image Super-Resolution`
(https://arxiv.org/pdf/1707.02921.pdf)

(single scale baseline-style model)
"""
class EDSR(object):

	def __init__(self,img_size=32,num_layers=32,feature_size=256,scale=2,output_channels=3):
		print("Building EDSR...")
		#Placeholder for image inputs
		self.input = x = tf.placeholder(tf.float32,[None,img_size,img_size,output_channels])
		#Placeholder for upscaled image ground-truth
		self.target = y = tf.placeholder(tf.float32,[None,img_size*scale,img_size*scale,output_channels])

		"""
		Preprocessing as mentioned in the paper, by subtracting the mean
		However, the subtract the mean of the entire dataset they use. As of
		now, I am subtracting the mean of each batch
		"""
		mean_x = tf.reduce_mean(self.input)
		image_input =x- mean_x
		mean_y = tf.reduce_mean(self.target)
		image_target =y- mean_y

		#One convolution before res blocks and to convert to required feature depth
		x = slim.conv2d(image_input,feature_size,[3,3])

		#Store the output of the first convolution to add later
		conv_1 = x

		"""
		This creates `num_layers` number of resBlocks
		a resBlock is defined in the paper as
		(excuse the ugly ASCII graph)
		x
		|\
		| \
		|  conv2d
		|  relu
		|  conv2d
		| /
		|/
		+ (addition here)
		|
		result
		"""

		"""
		Doing scaling here as mentioned in the paper:

		`we found that increasing the number of feature
		maps above a certain level would make the training procedure
		numerically unstable. A similar phenomenon was
		reported by Szegedy et al. We resolve this issue by
		adopting the residual scaling with factor 0.1. In each
		residual block, constant scaling layers are placed after the
		last convolution layers. These modules stabilize the training
		procedure greatly when using a large number of filters.
		In the test phase, this layer can be integrated into the previous
		convolution layer for the computational efficiency.'

		"""
		scaling_factor = 0.1

		#Add the residual blocks to the model
		for i in range(num_layers):
			x = utils.resBlock(x,feature_size,scale=scaling_factor)

		#One more convolution, and then we add the output of our first conv layer
		x = slim.conv2d(x,feature_size,[3,3])
		x += conv_1

		#Upsample output of the convolution
		x = utils.upsample(x,scale,feature_size,None)

		#One final convolution on the upsampling output
		self.out = output =x# slim.conv2d(x,output_channels,[3,3])

		self.loss = loss = tf.reduce_mean(tf.losses.absolute_difference(image_target,output))

		#Calculating Peak Signal-to-noise-ratio
		#Using equations from here: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio
		mse = tf.reduce_mean(tf.squared_difference(image_target,output))
		PSNR = tf.constant(255**2,dtype=tf.float32)/mse
		PSNR = tf.constant(10,dtype=tf.float32)*utils.log10(PSNR)

		#Scalar to keep track for loss
		tf.summary.scalar("loss",self.loss)
		tf.summary.scalar("PSNR",PSNR)
		#Image summaries for input, target, and output
		tf.summary.image("input_image",self.input+mean_x)
		tf.summary.image("target_image",self.target+mean_y)
		tf.summary.image("output_image",self.out+mean_x)

		#Tensorflow graph setup... session, saver, etc.
		self.sess = tf.Session()
		self.saver = tf.train.Saver()
		print("Done building!")

	"""
	Save the current state of the network to file
	"""
	def save(self,savedir='saved_models'):
		print("Saving...")
		self.saver.save(self.sess,savedir+"/model")
		print("Saved!")

	"""
	Resume network from previously saved weights
	"""
	def resume(self,savedir='saved_models'):
		print("Restoring...")
		self.saver.restore(self.sess,tf.train.latest_checkpoint(savedir))
		print("Restored!")

	"""
	Compute the output of this network given a specific input

	x: a tensor of shape [n,image_w,image_h,image_channels] where n is the number of images you have

	returns: a tensor of shape [n,image_width*2,img_height*2,channels] containing your super-resolution image(s)
	"""
	def predict(self,x):
		print("Predicting...")
		return self.sess.run(self.out,feed_dict={self.input:x})

	"""
	Function to setup your input data pipeline
	"""
	def set_data_fn(self,fn,args,test_set_fn=None,test_set_args=None):
		self.data = fn
		self.args = args
		self.test_data = test_set_fn
		self.test_args = test_set_args

	"""
	Train the neural network
	"""
	def train(self,iterations=1000,save_dir="saved_models"):
		#Removing previous save directory if there is one
		if os.path.exists(save_dir):
			shutil.rmtree(save_dir)
		#Make new save directory
		os.mkdir(save_dir)
		#Just a tf thing, to merge all summaries into one
		merged = tf.summary.merge_all()
		#Using adam optimizer as mentioned in the paper
		optimizer = tf.train.AdamOptimizer()
		#This is the train operation for our objective
		train_op = optimizer.minimize(self.loss)
		#Operation to initialize all variables
		init = tf.global_variables_initializer()
		print("Begin training...")
		with self.sess as sess:
			#Initialize all variables
			sess.run(init)
			test_exists = self.test_data
			#create summary writer for train
			train_writer = tf.summary.FileWriter(save_dir+"/train",sess.graph)

			#If we're using a test set, include another summary writer for that
			if test_exists:
				test_writer = tf.summary.FileWriter(save_dir+"/test",sess.graph)
				test_x,test_y = self.test_data(*self.test_args)
				test_feed = {self.input:test_x,self.target:test_y}

			#This is our training loop
			for i in tqdm(range(iterations)):
				#Use the data function we were passed to get a batch every iteration
				x,y = self.data(*self.args)
				#Create feed dictionary for the batch
				feed = {
					self.input:x,
					self.target:y
				}
				#Run the train op and calculate the train summary
				summary,_ = sess.run([merged,train_op],feed)
				#If we're testing, don't train on test set. But do calculate summary
				if test_exists:
					t_summary = sess.run(merged,test_feed)
					#Write test summary
					test_writer.add_summary(t_summary,i)
				#Write train summary for this step
				train_writer.add_summary(summary,i)
			#Save our trained model
			self.save()
<commit_msg>Clip output, cast image summaries to tf.uint8<commit_after>import tensorflow as tf
import tensorflow.contrib.slim as slim
import utils
import shutil
from tqdm import tqdm
import os

"""
An implementation of the neural network used for
super-resolution of images as described in:

`Enhanced Deep Residual Networks for Single Image Super-Resolution`
(https://arxiv.org/pdf/1707.02921.pdf)

(single scale baseline-style model)
"""
class EDSR(object):

	def __init__(self,img_size=32,num_layers=32,feature_size=256,scale=2,output_channels=3):
		print("Building EDSR...")
		#Placeholder for image inputs
		self.input = x = tf.placeholder(tf.float32,[None,img_size,img_size,output_channels])
		#Placeholder for upscaled image ground-truth
		self.target = y = tf.placeholder(tf.float32,[None,img_size*scale,img_size*scale,output_channels])

		"""
		Preprocessing as mentioned in the paper, by subtracting the mean
		However, the subtract the mean of the entire dataset they use. As of
		now, I am subtracting the mean of each batch
		"""
		mean_x = tf.reduce_mean(self.input)
		image_input =x- mean_x
		mean_y = tf.reduce_mean(self.target)
		image_target =y- mean_y

		#One convolution before res blocks and to convert to required feature depth
		x = slim.conv2d(image_input,feature_size,[3,3])

		#Store the output of the first convolution to add later
		conv_1 = x

		"""
		This creates `num_layers` number of resBlocks
		a resBlock is defined in the paper as
		(excuse the ugly ASCII graph)
		x
		|\
		| \
		|  conv2d
		|  relu
		|  conv2d
		| /
		|/
		+ (addition here)
		|
		result
		"""

		"""
		Doing scaling here as mentioned in the paper:

		`we found that increasing the number of feature
		maps above a certain level would make the training procedure
		numerically unstable. A similar phenomenon was
		reported by Szegedy et al. We resolve this issue by
		adopting the residual scaling with factor 0.1. In each
		residual block, constant scaling layers are placed after the
		last convolution layers. These modules stabilize the training
		procedure greatly when using a large number of filters.
		In the test phase, this layer can be integrated into the previous
		convolution layer for the computational efficiency.'

		"""
		scaling_factor = 0.1

		#Add the residual blocks to the model
		for i in range(num_layers):
			x = utils.resBlock(x,feature_size,scale=scaling_factor)

		#One more convolution, and then we add the output of our first conv layer
		x = slim.conv2d(x,feature_size,[3,3])
		x += conv_1

		#Upsample output of the convolution
		x = utils.upsample(x,scale,feature_size,None)

		#One final convolution on the upsampling output
		output =x# slim.conv2d(x,output_channels,[3,3])
		self.out = tf.clip_by_value(output+mean_x,0.0,255.0)

		self.loss = loss = tf.reduce_mean(tf.losses.absolute_difference(image_target,output))

		#Calculating Peak Signal-to-noise-ratio
		#Using equations from here: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio
		mse = tf.reduce_mean(tf.squared_difference(image_target,output))
		PSNR = tf.constant(255**2,dtype=tf.float32)/mse
		PSNR = tf.constant(10,dtype=tf.float32)*utils.log10(PSNR)

		#Scalar to keep track for loss
		tf.summary.scalar("loss",self.loss)
		tf.summary.scalar("PSNR",PSNR)
		#Image summaries for input, target, and output
		tf.summary.image("input_image",tf.cast(self.input,tf.uint8))
		tf.summary.image("target_image",tf.cast(self.target,tf.uint8))
		tf.summary.image("output_image",tf.cast(self.out,tf.uint8))

		#Tensorflow graph setup... session, saver, etc.
		self.sess = tf.Session()
		self.saver = tf.train.Saver()
		print("Done building!")

	"""
	Save the current state of the network to file
	"""
	def save(self,savedir='saved_models'):
		print("Saving...")
		self.saver.save(self.sess,savedir+"/model")
		print("Saved!")

	"""
	Resume network from previously saved weights
	"""
	def resume(self,savedir='saved_models'):
		print("Restoring...")
		self.saver.restore(self.sess,tf.train.latest_checkpoint(savedir))
		print("Restored!")

	"""
	Compute the output of this network given a specific input

	x: a tensor of shape [n,image_w,image_h,image_channels] where n is the number of images you have

	returns: a tensor of shape [n,image_width*2,img_height*2,channels] containing your super-resolution image(s)
	"""
	def predict(self,x):
		print("Predicting...")
		return self.sess.run(self.out,feed_dict={self.input:x})

	"""
	Function to setup your input data pipeline
	"""
	def set_data_fn(self,fn,args,test_set_fn=None,test_set_args=None):
		self.data = fn
		self.args = args
		self.test_data = test_set_fn
		self.test_args = test_set_args

	"""
	Train the neural network
	"""
	def train(self,iterations=1000,save_dir="saved_models"):
		#Removing previous save directory if there is one
		if os.path.exists(save_dir):
			shutil.rmtree(save_dir)
		#Make new save directory
		os.mkdir(save_dir)
		#Just a tf thing, to merge all summaries into one
		merged = tf.summary.merge_all()
		#Using adam optimizer as mentioned in the paper
		optimizer = tf.train.AdamOptimizer()
		#This is the train operation for our objective
		train_op = optimizer.minimize(self.loss)
		#Operation to initialize all variables
		init = tf.global_variables_initializer()
		print("Begin training...")
		with self.sess as sess:
			#Initialize all variables
			sess.run(init)
			test_exists = self.test_data
			#create summary writer for train
			train_writer = tf.summary.FileWriter(save_dir+"/train",sess.graph)

			#If we're using a test set, include another summary writer for that
			if test_exists:
				test_writer = tf.summary.FileWriter(save_dir+"/test",sess.graph)
				test_x,test_y = self.test_data(*self.test_args)
				test_feed = {self.input:test_x,self.target:test_y}

			#This is our training loop
			for i in tqdm(range(iterations)):
				#Use the data function we were passed to get a batch every iteration
				x,y = self.data(*self.args)
				#Create feed dictionary for the batch
				feed = {
					self.input:x,
					self.target:y
				}
				#Run the train op and calculate the train summary
				summary,_ = sess.run([merged,train_op],feed)
				#If we're testing, don't train on test set. But do calculate summary
				if test_exists:
					t_summary = sess.run(merged,test_feed)
					#Write test summary
					test_writer.add_summary(t_summary,i)
				#Write train summary for this step
				train_writer.add_summary(summary,i)
			#Save our trained model
			self.save()
<|endoftext|>