<commit_before># -*- coding: utf-8 -*-

## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Session management adapted from mod_python Session class.

Just use L{get_session} to obtain a session object (with a dictionary
interface, which will let you store permanent information).
"""

from invenio.webinterface_handler_wsgi_utils import add_cookie, Cookie, get_cookie

import cPickle
import time
import random
import re
import sys
import os
from thread import get_ident
if sys.hexversion < 0x2060000:
    from md5 import md5
else:
    from hashlib import md5

from invenio.dbquery import run_sql, blob_to_string
from invenio.config import CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER, \
    CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT
from invenio.websession_config import CFG_WEBSESSION_COOKIE_NAME, \
    CFG_WEBSESSION_ONE_DAY, CFG_WEBSESSION_CLEANUP_CHANCE, \
    CFG_WEBSESSION_ENABLE_LOCKING

def get_session(req, sid=None):
    """
    Obtain a session.

    If the session has already been created for the current request,
    returns the already existing session.

    @param req: the mod_python request object.
    @type req: mod_python request object
    @param sid: the session identifier of an already existing session.
    @type sid: 32 hexadecimal string
    @return: the session.
    @rtype: InvenioSession
    @raise ValueError: if C{sid} is provided and it doesn't correspond to a
        valid session.
    @note: if the session has already been retrieved once within the current
        request handling, the same session object will be returned and, if
        provided, the C{sid} parameter will be ignored.
    """
    if not hasattr(req, '_session'):
        req._session = InvenioSession(req, sid)
    return req._session

class InvenioSession(dict):
    """
    This class implements a Session handling based on MySQL.

    @param req: the mod_python request object.
    @type req: mod_python request object
    @param sid: the session identifier if already known
    @type sid: 32 hexadecimal string
    @ivar _remember_me: if the session cookie should last one day or until
        the browser is closed.
    @type _remember_me: bool

    @note: The code is heavily based on ModPython 3.3.1 DBMSession
        implementation.
    @note: This class implements IP verification to prevent basic cookie
        stealing.
    @raise ValueError: if C{sid} is provided and correspond to a broken
        session.
    """

    def __init__(self, req, sid=None):
        self._remember_me = False
        self._req, self._sid, self._secret = req, sid, None
        self._lock = CFG_WEBSESSION_ENABLE_LOCKING
        self._new = 1
        self._created = 0
        self._accessed = 0
        self._timeout = 0
        self._locked = 0
        self._invalid = 0
        self._http_ip = None
        self._https_ip = None

        dict.__init__(self)

        if not self._sid:
            # check to see if cookie exists
            cookie = get_cookie(req, CFG_WEBSESSION_COOKIE_NAME)
            if cookie:
                self._sid = cookie.value

        if self._sid:
            if not _check_sid(self._sid):
                if sid:
                    # Supplied explicitly by user of the class,
                    # raise an exception and make the user code
                    # deal with it.
                    raise ValueError("Invalid Session ID: sid=%s" % sid)
                else:
                    # Derived from the cookie sent by browser,
                    # wipe it out so it gets replaced with a
                    # correct value.
                    self._sid = None

        if self._sid:
            # attempt to load ourselves
            self.lock()
            if self.load():
                self._new = 0

        if self._new:
            # make a new session
            if self._sid:
                self.unlock() # unlock old sid
            self._sid = _new_sid(self._req)
            self.lock()                 # lock new sid
            remote_ip = self._req.remote_ip
            if self._req.is_https():
                self._https_ip = remote_ip
            else:
                self._http_ip = remote_ip
            add_cookie(self._req, self.make_cookie())
            self._created = time.time()
            self._timeout = CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT * \
                CFG_WEBSESSION_ONE_DAY

        self._accessed = time.time()

        # need cleanup?
        if random.randint(1, CFG_WEBSESSION_CLEANUP_CHANCE) == 1:
            self.cleanup()

    def set_remember_me(self, remember_me=True):
        """
        Set/Unset the L{_remember_me} flag.

        @param remember_me: True if the session cookie should last one day or
            until the browser is closed.
        @type remember_me: bool
        """
        self._remember_me = remember_me
        if remember_me:
            self.set_timeout(CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER *
                CFG_WEBSESSION_ONE_DAY)
        else:
            self.set_timeout(CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT *
                CFG_WEBSESSION_ONE_DAY)
        add_cookie(self._req, self.make_cookie())

    def load(self):
        """
        Load the session from the database.
        @return: 1 in case of success, 0 otherwise.
        @rtype: integer
        """
        session_dict = None
        invalid = False
        res = run_sql("SELECT session_object FROM session "
                        "WHERE session_key=%s", (self._sid, ))
        if res:
            session_dict = cPickle.loads(blob_to_string(res[0][0]))
            remote_ip = self._req.remote_ip
            if self._req.is_https():
                if session_dict['_https_ip'] is not None and \
                        session_dict['_https_ip'] != remote_ip:
                    invalid = True
                else:
                    session_dict['_https_ip'] = remote_ip
            else:
                if session_dict['_http_ip'] is not None and \
                        session_dict['_http_ip'] != remote_ip:
                    invalid = True
                else:
                    session_dict['_http_ip'] = remote_ip

        if session_dict is None:
            return 0

        if invalid:
            return 0

        if (time.time() - session_dict["_accessed"]) > \
                session_dict["_timeout"]:
            return 0

        self._created  = session_dict["_created"]
        self._accessed = session_dict["_accessed"]
        self._timeout  = session_dict["_timeout"]
        self._remember_me = session_dict["_remember_me"]
        self.update(session_dict["_data"])
        return 1

    def save(self):
        """
        Save the session to the database.
        """
        if not self._invalid:
            session_dict = {"_data" : self.copy(),
                    "_created" : self._created,
                    "_accessed": self._accessed,
                    "_timeout" : self._timeout,
                    "_http_ip" : self._http_ip,
                    "_https_ip" : self._https_ip,
                    "_remember_me" : self._remember_me
            }
            session_key = self._sid
            session_object = cPickle.dumps(session_dict, -1)
            session_expiry = time.time() + self._timeout + \
                CFG_WEBSESSION_ONE_DAY
            uid = self.get('uid', -1)
            run_sql("""
                INSERT session(
                    session_key,
                    session_expiry,
                    session_object,
                    uid
                ) VALUE(%s,
                    %s,
                    %s,
                    %s
                ) ON DUPLICATE KEY UPDATE
                    session_expiry=%s,
                    session_object=%s,
                    uid=%s
            """, (session_key, session_expiry, session_object, uid,
                session_expiry, session_object, uid))

    def delete(self):
        """
        Delete the session.
        """
        run_sql("DELETE FROM session WHERE session_key=%s", (self._sid, ))
        self.clear()

    def invalidate(self):
        """
        Declare the session as invalid.
        """
        cookie = self.make_cookie()
        cookie.expires = 0
        add_cookie(self._req, cookie)
        self.delete()
        self._invalid = 1
        if hasattr(self._req, '_session'):
            delattr(self._req, '_session')

    def make_cookie(self):
        """
        Reimplementation of L{BaseSession.make_cookie} method, that also
        consider the L{_remember_me} flag

        @return: a session cookie.
        @rtpye: {mod_python.Cookie.Cookie}
        """
        cookie = Cookie(CFG_WEBSESSION_COOKIE_NAME, self._sid)
        cookie.path = '/'

        if self._remember_me:
            cookie.expires = time.time() + self._timeout

        return cookie

    def initial_http_ip(self):
        """
        @return: the initial ip addressed for the HTTP protocol for which this
        session was issued.
        @rtype: string
        @note: it returns None if this session has always been used through
            HTTPS requests.
        """
        return self._http_ip

    def initial_https_ip(self):
        """
        @return: the initial ip addressed for the HTTPS protocol for which this
        session was issued.
        @rtype: string
        @note: it returns None if this session has always been used through
            HTTP requests.
        """
        return self._https_ip

    def lock(self):
        """
        Lock the session.
        """
        if self._lock:
            self._locked = 1

    def unlock(self):
        """
        Unlock the session.
        """
        if self._lock and self._locked:
            self._locked = 0

    def is_new(self):
        """
        @return: True if the session has just been created.
        @rtype: bool
        """
        return not not self._new

    def sid(self):
        """
        @return: the session identifier.
        @rtype: 32 hexadecimal string
        """
        return self._sid

    def created(self):
        """
        @return: the UNIX timestamp for when the session has been created.
        @rtype: double
        """
        return self._created

    def last_accessed(self):
        """
        @return: the UNIX timestamp for when the session has been last
            accessed.
        @rtype: double
        """
        return self._accessed

    def timeout(self):
        """
        @return: the number of seconds from the last accessed timestamp,
            after which the session is invalid.
        @rtype: double
        """
        return self._timeout

    def set_timeout(self, secs):
        """
        Set the number of seconds from the last accessed timestamp,
            after which the session is invalid.
        @param secs: the number of seconds.
        @type secs: double
        """
        self._timeout = secs

    def cleanup(self):
        """
        Perform the database session cleanup.
        """
        def session_cleanup():
            """
            Session cleanup procedure which to be executed at the end
            of the request handling.
            """
            run_sql("""
                DELETE FROM session
                WHERE session_expiry<=UNIX_TIMESTAMP()
            """)
        self._req.register_cleanup(session_cleanup)
        self._req.log_error("InvenioSession: registered database cleanup.",
                            APLOG_NOTICE)

    def __del__(self):
        self.unlock()

def _unlock_session_cleanup(session):
    """
    Auxliary function to unlock a session.
    """
    session.unlock()

def _init_rnd():
    """
    Initialize random number generators.
    This is key in multithreaded env, see Python docs for random.
    @return: the generators.
    @rtype: list of generators
    """

    # query max number of threads
    gennum = 10

    # make generators
    # this bit is from Python lib reference
    random_generator = random.Random(time.time())
    result = [random_generator]
    for dummy in range(gennum - 1):
        laststate = random_generator.getstate()
        random_generator = random.Random()
        random_generator.setstate(laststate)
        random_generator.jumpahead(1000000)
        result.append(random_generator)

    return result

_RANDOM_GENERATORS = _init_rnd()
_RANDOM_ITERATOR = iter(_RANDOM_GENERATORS)

def _get_generator():
    """
    get rnd_iter.next(), or start over
    if we reached the end of it
    @return: the next random number.
    @rtype: double
    """
    global _RANDOM_ITERATOR
    try:
        return _RANDOM_ITERATOR.next()
    except StopIteration:
        # the small potential for two threads doing this
        # seems does not warrant use of a lock
        _RANDOM_ITERATOR = iter(_RANDOM_GENERATORS)
        return _RANDOM_ITERATOR.next()

_RE_VALIDATE_SID = re.compile('[0-9a-f]{32}$')
def _check_sid(sid):
    """
    Check the validity of the session identifier.
    The sid must be 32 characters long, and consisting of the characters
    0-9 and a-f.

    The sid may be passed in a cookie from the client and as such
    should not be trusted. This is particularly important in
    FileSession, where the session filename is derived from the sid.
    A sid containing '/' or '.' characters could result in a directory
    traversal attack

    @param sid: the session identifier.
    @type sid: string
    @return: True if the session identifier is valid.
    @rtype: bool
    """
    return not not _RE_VALIDATE_SID.match(sid)

def _new_sid(req):
    """
    Make a number based on current time, pid, remote ip
    and two random ints, then hash with md5. This should
    be fairly unique and very difficult to guess.

    @param req: the mod_python request object.
    @type req: mod_python request object.
    @return: the session identifier.
    @rtype: 32 hexadecimal string

    @warning: The current implementation of _new_sid returns an
        md5 hexdigest string. To avoid a possible directory traversal
        attack in FileSession the sid is validated using
        the _check_sid() method and the compiled regex
        validate_sid_re. The sid will be accepted only if len(sid) == 32
        and it only contains the characters 0-9 and a-f.

        If you change this implementation of _new_sid, make sure to also
        change the validation scheme, as well as the test_Session_illegal_sid()
        unit test in test/test.py.
    """

    the_time = long(time.time()*10000)
    pid = os.getpid()
    random_generator = _get_generator()
    rnd1 = random_generator.randint(0, 999999999)
    rnd2 = random_generator.randint(0, 999999999)
    remote_ip = req.remote_ip

    return md5("%d%d%d%d%s" % (
        the_time,
        pid,
        rnd1,
        rnd2,
        remote_ip)
    ).hexdigest()
<commit_msg>WebSession: fix APLOG_NOTICE bug (API change)<commit_after># -*- coding: utf-8 -*-

## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Session management adapted from mod_python Session class.

Just use L{get_session} to obtain a session object (with a dictionary
interface, which will let you store permanent information).
"""

from invenio.webinterface_handler_wsgi_utils import add_cookie, Cookie, get_cookie

import cPickle
import time
import random
import re
import sys
import os
from thread import get_ident
if sys.hexversion < 0x2060000:
    from md5 import md5
else:
    from hashlib import md5

from invenio.dbquery import run_sql, blob_to_string
from invenio.config import CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER, \
    CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT
from invenio.websession_config import CFG_WEBSESSION_COOKIE_NAME, \
    CFG_WEBSESSION_ONE_DAY, CFG_WEBSESSION_CLEANUP_CHANCE, \
    CFG_WEBSESSION_ENABLE_LOCKING

def get_session(req, sid=None):
    """
    Obtain a session.

    If the session has already been created for the current request,
    returns the already existing session.

    @param req: the mod_python request object.
    @type req: mod_python request object
    @param sid: the session identifier of an already existing session.
    @type sid: 32 hexadecimal string
    @return: the session.
    @rtype: InvenioSession
    @raise ValueError: if C{sid} is provided and it doesn't correspond to a
        valid session.
    @note: if the session has already been retrieved once within the current
        request handling, the same session object will be returned and, if
        provided, the C{sid} parameter will be ignored.
    """
    if not hasattr(req, '_session'):
        req._session = InvenioSession(req, sid)
    return req._session

class InvenioSession(dict):
    """
    This class implements a Session handling based on MySQL.

    @param req: the mod_python request object.
    @type req: mod_python request object
    @param sid: the session identifier if already known
    @type sid: 32 hexadecimal string
    @ivar _remember_me: if the session cookie should last one day or until
        the browser is closed.
    @type _remember_me: bool

    @note: The code is heavily based on ModPython 3.3.1 DBMSession
        implementation.
    @note: This class implements IP verification to prevent basic cookie
        stealing.
    @raise ValueError: if C{sid} is provided and correspond to a broken
        session.
    """

    def __init__(self, req, sid=None):
        self._remember_me = False
        self._req, self._sid, self._secret = req, sid, None
        self._lock = CFG_WEBSESSION_ENABLE_LOCKING
        self._new = 1
        self._created = 0
        self._accessed = 0
        self._timeout = 0
        self._locked = 0
        self._invalid = 0
        self._http_ip = None
        self._https_ip = None

        dict.__init__(self)

        if not self._sid:
            # check to see if cookie exists
            cookie = get_cookie(req, CFG_WEBSESSION_COOKIE_NAME)
            if cookie:
                self._sid = cookie.value

        if self._sid:
            if not _check_sid(self._sid):
                if sid:
                    # Supplied explicitly by user of the class,
                    # raise an exception and make the user code
                    # deal with it.
                    raise ValueError("Invalid Session ID: sid=%s" % sid)
                else:
                    # Derived from the cookie sent by browser,
                    # wipe it out so it gets replaced with a
                    # correct value.
                    self._sid = None

        if self._sid:
            # attempt to load ourselves
            self.lock()
            if self.load():
                self._new = 0

        if self._new:
            # make a new session
            if self._sid:
                self.unlock() # unlock old sid
            self._sid = _new_sid(self._req)
            self.lock()                 # lock new sid
            remote_ip = self._req.remote_ip
            if self._req.is_https():
                self._https_ip = remote_ip
            else:
                self._http_ip = remote_ip
            add_cookie(self._req, self.make_cookie())
            self._created = time.time()
            self._timeout = CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT * \
                CFG_WEBSESSION_ONE_DAY

        self._accessed = time.time()

        # need cleanup?
        if random.randint(1, CFG_WEBSESSION_CLEANUP_CHANCE) == 1:
            self.cleanup()

    def set_remember_me(self, remember_me=True):
        """
        Set/Unset the L{_remember_me} flag.

        @param remember_me: True if the session cookie should last one day or
            until the browser is closed.
        @type remember_me: bool
        """
        self._remember_me = remember_me
        if remember_me:
            self.set_timeout(CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER *
                CFG_WEBSESSION_ONE_DAY)
        else:
            self.set_timeout(CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT *
                CFG_WEBSESSION_ONE_DAY)
        add_cookie(self._req, self.make_cookie())

    def load(self):
        """
        Load the session from the database.
        @return: 1 in case of success, 0 otherwise.
        @rtype: integer
        """
        session_dict = None
        invalid = False
        res = run_sql("SELECT session_object FROM session "
                        "WHERE session_key=%s", (self._sid, ))
        if res:
            session_dict = cPickle.loads(blob_to_string(res[0][0]))
            remote_ip = self._req.remote_ip
            if self._req.is_https():
                if session_dict['_https_ip'] is not None and \
                        session_dict['_https_ip'] != remote_ip:
                    invalid = True
                else:
                    session_dict['_https_ip'] = remote_ip
            else:
                if session_dict['_http_ip'] is not None and \
                        session_dict['_http_ip'] != remote_ip:
                    invalid = True
                else:
                    session_dict['_http_ip'] = remote_ip

        if session_dict is None:
            return 0

        if invalid:
            return 0

        if (time.time() - session_dict["_accessed"]) > \
                session_dict["_timeout"]:
            return 0

        self._created  = session_dict["_created"]
        self._accessed = session_dict["_accessed"]
        self._timeout  = session_dict["_timeout"]
        self._remember_me = session_dict["_remember_me"]
        self.update(session_dict["_data"])
        return 1

    def save(self):
        """
        Save the session to the database.
        """
        if not self._invalid:
            session_dict = {"_data" : self.copy(),
                    "_created" : self._created,
                    "_accessed": self._accessed,
                    "_timeout" : self._timeout,
                    "_http_ip" : self._http_ip,
                    "_https_ip" : self._https_ip,
                    "_remember_me" : self._remember_me
            }
            session_key = self._sid
            session_object = cPickle.dumps(session_dict, -1)
            session_expiry = time.time() + self._timeout + \
                CFG_WEBSESSION_ONE_DAY
            uid = self.get('uid', -1)
            run_sql("""
                INSERT session(
                    session_key,
                    session_expiry,
                    session_object,
                    uid
                ) VALUE(%s,
                    %s,
                    %s,
                    %s
                ) ON DUPLICATE KEY UPDATE
                    session_expiry=%s,
                    session_object=%s,
                    uid=%s
            """, (session_key, session_expiry, session_object, uid,
                session_expiry, session_object, uid))

    def delete(self):
        """
        Delete the session.
        """
        run_sql("DELETE FROM session WHERE session_key=%s", (self._sid, ))
        self.clear()

    def invalidate(self):
        """
        Declare the session as invalid.
        """
        cookie = self.make_cookie()
        cookie.expires = 0
        add_cookie(self._req, cookie)
        self.delete()
        self._invalid = 1
        if hasattr(self._req, '_session'):
            delattr(self._req, '_session')

    def make_cookie(self):
        """
        Reimplementation of L{BaseSession.make_cookie} method, that also
        consider the L{_remember_me} flag

        @return: a session cookie.
        @rtpye: {mod_python.Cookie.Cookie}
        """
        cookie = Cookie(CFG_WEBSESSION_COOKIE_NAME, self._sid)
        cookie.path = '/'

        if self._remember_me:
            cookie.expires = time.time() + self._timeout

        return cookie

    def initial_http_ip(self):
        """
        @return: the initial ip addressed for the HTTP protocol for which this
        session was issued.
        @rtype: string
        @note: it returns None if this session has always been used through
            HTTPS requests.
        """
        return self._http_ip

    def initial_https_ip(self):
        """
        @return: the initial ip addressed for the HTTPS protocol for which this
        session was issued.
        @rtype: string
        @note: it returns None if this session has always been used through
            HTTP requests.
        """
        return self._https_ip

    def lock(self):
        """
        Lock the session.
        """
        if self._lock:
            self._locked = 1

    def unlock(self):
        """
        Unlock the session.
        """
        if self._lock and self._locked:
            self._locked = 0

    def is_new(self):
        """
        @return: True if the session has just been created.
        @rtype: bool
        """
        return not not self._new

    def sid(self):
        """
        @return: the session identifier.
        @rtype: 32 hexadecimal string
        """
        return self._sid

    def created(self):
        """
        @return: the UNIX timestamp for when the session has been created.
        @rtype: double
        """
        return self._created

    def last_accessed(self):
        """
        @return: the UNIX timestamp for when the session has been last
            accessed.
        @rtype: double
        """
        return self._accessed

    def timeout(self):
        """
        @return: the number of seconds from the last accessed timestamp,
            after which the session is invalid.
        @rtype: double
        """
        return self._timeout

    def set_timeout(self, secs):
        """
        Set the number of seconds from the last accessed timestamp,
            after which the session is invalid.
        @param secs: the number of seconds.
        @type secs: double
        """
        self._timeout = secs

    def cleanup(self):
        """
        Perform the database session cleanup.
        """
        def session_cleanup():
            """
            Session cleanup procedure which to be executed at the end
            of the request handling.
            """
            run_sql("""
                DELETE FROM session
                WHERE session_expiry<=UNIX_TIMESTAMP()
            """)
        self._req.register_cleanup(session_cleanup)
        self._req.log_error("InvenioSession: registered database cleanup.")

    def __del__(self):
        self.unlock()

def _unlock_session_cleanup(session):
    """
    Auxliary function to unlock a session.
    """
    session.unlock()

def _init_rnd():
    """
    Initialize random number generators.
    This is key in multithreaded env, see Python docs for random.
    @return: the generators.
    @rtype: list of generators
    """

    # query max number of threads
    gennum = 10

    # make generators
    # this bit is from Python lib reference
    random_generator = random.Random(time.time())
    result = [random_generator]
    for dummy in range(gennum - 1):
        laststate = random_generator.getstate()
        random_generator = random.Random()
        random_generator.setstate(laststate)
        random_generator.jumpahead(1000000)
        result.append(random_generator)

    return result

_RANDOM_GENERATORS = _init_rnd()
_RANDOM_ITERATOR = iter(_RANDOM_GENERATORS)

def _get_generator():
    """
    get rnd_iter.next(), or start over
    if we reached the end of it
    @return: the next random number.
    @rtype: double
    """
    global _RANDOM_ITERATOR
    try:
        return _RANDOM_ITERATOR.next()
    except StopIteration:
        # the small potential for two threads doing this
        # seems does not warrant use of a lock
        _RANDOM_ITERATOR = iter(_RANDOM_GENERATORS)
        return _RANDOM_ITERATOR.next()

_RE_VALIDATE_SID = re.compile('[0-9a-f]{32}$')
def _check_sid(sid):
    """
    Check the validity of the session identifier.
    The sid must be 32 characters long, and consisting of the characters
    0-9 and a-f.

    The sid may be passed in a cookie from the client and as such
    should not be trusted. This is particularly important in
    FileSession, where the session filename is derived from the sid.
    A sid containing '/' or '.' characters could result in a directory
    traversal attack

    @param sid: the session identifier.
    @type sid: string
    @return: True if the session identifier is valid.
    @rtype: bool
    """
    return not not _RE_VALIDATE_SID.match(sid)

def _new_sid(req):
    """
    Make a number based on current time, pid, remote ip
    and two random ints, then hash with md5. This should
    be fairly unique and very difficult to guess.

    @param req: the mod_python request object.
    @type req: mod_python request object.
    @return: the session identifier.
    @rtype: 32 hexadecimal string

    @warning: The current implementation of _new_sid returns an
        md5 hexdigest string. To avoid a possible directory traversal
        attack in FileSession the sid is validated using
        the _check_sid() method and the compiled regex
        validate_sid_re. The sid will be accepted only if len(sid) == 32
        and it only contains the characters 0-9 and a-f.

        If you change this implementation of _new_sid, make sure to also
        change the validation scheme, as well as the test_Session_illegal_sid()
        unit test in test/test.py.
    """

    the_time = long(time.time()*10000)
    pid = os.getpid()
    random_generator = _get_generator()
    rnd1 = random_generator.randint(0, 999999999)
    rnd2 = random_generator.randint(0, 999999999)
    remote_ip = req.remote_ip

    return md5("%d%d%d%d%s" % (
        the_time,
        pid,
        rnd1,
        rnd2,
        remote_ip)
    ).hexdigest()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import wmi
import psutil
import win32com.client
from win32gui import GetWindowText, GetForegroundWindow

shell = win32com.client.Dispatch('WScript.Shell')

current_pid = GetForegroundWindow()
pid_ids = [p.id for p in psutil.process_iter() if 'firefox' in p.name()][-1]
if pid_ids:
   pid = pid_ids[-1] # last
   shell.AppActivate(pid)
   shell.SendKeys('^{F4}') # CTRL + F4
   print 'Close Firefox tab ID: %s' % pid
else:
   print 'No Firefox'

shell.AppActivate(current_pid)

#shell.Run('firefox')
#shell.AppActivate('firefox')
#argv = sys.argv
#if len(argv) == 2:
#    mode = 'print'
#    print 'Print mode'
#else:
#    mode = 'delete'
#    print 'Delete mode'
#c = wmi.WMI()
#for process in c.Win32_Process():
#    if process.Name.lower() == 'firefox.exe':
#        # Read parameters:
#        pid = process.ProcessId
#        shell.AppActivate(pid)
#        caption = GetWindowText(GetForegroundWindow())
#
#        #print 'Firefox process ID: %s [%s]' % (pid, caption)
#        if 'Nuova scheda' in caption or 'Mozilla Firefox' in caption:
#            shell.SendKeys('^{F4}') # CTRL + F4
#            print 'Close Firefox tab ID: %s' % pid
#            break
#        #new_form.append(pid)
#        #print 'Firefox PID: %s [%s]' % (pid, caption)

#if mode == 'delete' and new_form:
#    print 'Firefox total ID: %s' % (new_form, )
#    for firefox_id in new_form:
#        shell.AppActivate(firefox_id)
#        shell.SendKeys('^{F4}') # CTRL + F4
#        print 'Close Firefox tab ID: %s' % firefox_id
#else:
#    print 'No Firefox open'

# Activate before window:
#shell.AppActivate(current_pid)

#print 'Windows active ID: %s' % GetWindowText(GetForegroundWindow())
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>fast git<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import wmi
import psutil
import win32com.client
from win32gui import GetWindowText, GetForegroundWindow

shell = win32com.client.Dispatch('WScript.Shell')

current_pid = GetForegroundWindow()
pid_ids = [p.pid for p in psutil.process_iter() if 'firefox' in p.name()][-1]
if pid_ids:
   pid = pid_ids[-1] # last
   shell.AppActivate(pid)
   shell.SendKeys('^{F4}') # CTRL + F4
   print 'Close Firefox tab ID: %s' % pid
else:
   print 'No Firefox'

shell.AppActivate(current_pid)

#shell.Run('firefox')
#shell.AppActivate('firefox')
#argv = sys.argv
#if len(argv) == 2:
#    mode = 'print'
#    print 'Print mode'
#else:
#    mode = 'delete'
#    print 'Delete mode'
#c = wmi.WMI()
#for process in c.Win32_Process():
#    if process.Name.lower() == 'firefox.exe':
#        # Read parameters:
#        pid = process.ProcessId
#        shell.AppActivate(pid)
#        caption = GetWindowText(GetForegroundWindow())
#
#        #print 'Firefox process ID: %s [%s]' % (pid, caption)
#        if 'Nuova scheda' in caption or 'Mozilla Firefox' in caption:
#            shell.SendKeys('^{F4}') # CTRL + F4
#            print 'Close Firefox tab ID: %s' % pid
#            break
#        #new_form.append(pid)
#        #print 'Firefox PID: %s [%s]' % (pid, caption)

#if mode == 'delete' and new_form:
#    print 'Firefox total ID: %s' % (new_form, )
#    for firefox_id in new_form:
#        shell.AppActivate(firefox_id)
#        shell.SendKeys('^{F4}') # CTRL + F4
#        print 'Close Firefox tab ID: %s' % firefox_id
#else:
#    print 'No Firefox open'

# Activate before window:
#shell.AppActivate(current_pid)

#print 'Windows active ID: %s' % GetWindowText(GetForegroundWindow())
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>from .estimator_base import H2OEstimator
from h2o.connection import H2OConnection

class H2OGeneralizedLinearEstimator(H2OEstimator):
  """Build a Generalized Linear Model
    Fit a generalized linear model, specified by a response variable, a set of predictors,
    and a description of the error distribution.

    Parameters
    ----------
      model_id : str, optional
        The unique id assigned to the resulting model. If none is given, an id will
        automatically be generated.

      max_iterations : int
        A non-negative integer specifying the maximum number of iterations.

      beta_epsilon : int
        A non-negative number specifying the magnitude of the maximum difference between
        the coefficient estimates from successive iterations. Defines the convergence
        criterion.

      solver : str
        A character string specifying the solver used: IRLSM (supports more features),
        L_BFGS (scales better for datasets with many columns)

      standardize : bool
        Indicates whether the numeric predictors should be standardized to have a mean of
        0 and a variance of 1 prior to training the models.

      family : str
        A character string specifying the distribution of the model:
        gaussian, binomial, multinomial, poisson, gamma, tweedie.

      link : str
        A character string specifying the link function. The default is the canonical
        link for the family. The supported links for each of the family specifications are
        "gaussian" - "identity", "log", "inverse"
        "binomial" - "logit", "log"
        "multinomial" - "multinomial"
        "poisson" - "log", "identity"
        "gamma" - "inverse", "log", "identity"
        "tweedie" - "tweedie"

      tweedie_variance_power : int
        numeric specifying the power for the variance function when family = "tweedie".

      tweedie_link_power : int
        A numeric specifying the power for the link function when family = "tweedie".

      alpha : float
        A numeric in [0, 1] specifying the elastic-net mixing parameter.
        The elastic-net penalty is defined to be
        eqn{P(\alpha,\beta) = (1-\alpha)/2||\beta||_2^2 +
        \alpha||\beta||_1 = \sum_j [(1-\alpha)/2 \beta_j^2 + \alpha|\beta_j|],
        making alpha = 1 the lasso penalty and alpha = 0 the ridge penalty.

      Lambda : float
        A non-negative shrinkage parameter for the elastic-net, which multiplies
        \eqn{P(\alpha,\beta) in the objective function.
        When Lambda = 0, no elastic-net penalty is applied and ordinary generalized linear
        models are fit.

      prior : float, optional
        A numeric specifying the prior probability of class 1 in the response when
        family = "binomial". The default prior is the observational frequency of class 1.
        Must be from (0,1) exclusive range or None (no prior).

      lambda_search : bool
        A logical value indicating whether to conduct a search over the space of lambda
        values starting from the lambda max, given lambda is interpreted as lambda minself.

      nlambdas : int
        The number of lambda values to use when lambda_search = TRUE.

      lambda_min_ratio : float
        Smallest value for lambda as a fraction of lambda.max. By default if the number of
        observations is greater than the the number of variables then
        lambda_min_ratio = 0.0001; if the number of observations is less than the number
        of variables then lambda_min_ratio = 0.01.

      beta_constraints : H2OFrame
        A data.frame or H2OParsedData object with the columns
        ["names", "lower_bounds", "upper_bounds", "beta_given"],
        where each row corresponds to a predictor in the GLM.
        "names" contains the predictor names, "lower"/"upper_bounds",
        are the lower and upper bounds of beta, and "beta_given" is some supplied starting
        values.

      nfolds : int, optional
        Number of folds for cross-validation. If nfolds >= 2, then validation must
        remain empty.

      fold_assignment : str
        Cross-validation fold assignment scheme, if fold_column is not specified Must be
        "AUTO", "Random" or "Modulo"

      keep_cross_validation_predictions : bool
        Whether to keep the predictions of the cross-validation models

      keep_cross_validation_fold_assignment : bool
        Whether to keep the cross-validation fold assignment.

      intercept : bool
        Logical, include constant term (intercept) in the model

      max_active_predictors : int, optional
        Convergence criteria for number of predictors when using L1 penalty.

      missing_values_handling : str
        A character string specifying how to handle missing value:
        "MeanImputation","Skip".

      interactions : list, optional
        A list of column names to interact. All pairwise combinations of columns will be
        interacted.
      max_runtime_secs: int, optional
        Maximum allowed runtime, model will stop running after reaching the limit and return whatever result it has at the moment.

    Returns
    -------
      A subclass of ModelBase is returned. The specific subclass depends on the machine
      learning task at hand (if it's binomial classification, then an H2OBinomialModel
      is returned, if it's regression then a H2ORegressionModel is returned). The default
      print-out of the models is shown, but further GLM-specifc information can be
      queried out of the object. Upon completion of the GLM, the resulting object has
      coefficients, normalized coefficients, residual/null deviance, aic, and a host of
      model metrics including MSE, AUC (for logistic regression), degrees of freedom, and
      confusion matrices.
    """
  def __init__(self, model_id=None, max_iterations=None, beta_epsilon=None, solver=None,
               standardize=None, family=None, link=None, tweedie_variance_power=None,
               tweedie_link_power=None, alpha=None, prior=None, lambda_search=None,
               nlambdas=None, lambda_min_ratio=None, beta_constraints=None, nfolds=None,
               fold_assignment=None, keep_cross_validation_predictions=None,
               keep_cross_validation_fold_assignment=None,
               intercept=None, Lambda=None, max_active_predictors=None, checkpoint=None,
               objective_epsilon=None, gradient_epsilon=None, non_negative=False,
               compute_p_values=False, remove_collinear_columns=False,
               missing_values_handling=None, interactions=None):
    super(H2OGeneralizedLinearEstimator, self).__init__()
    self._parms = locals()
    self._parms = {k: v for k, v in self._parms.items() if k != "self"}
    self._parms["lambda"] = self._parms.pop("Lambda")

  @property
  def max_iterations(self):
    return self._parms["max_iterations"]

  @max_iterations.setter
  def max_iterations(self, value):
    self._parms["max_iterations"] = value

  @property
  def beta_epsilon(self):
    return self._parms["beta_epsilon"]

  @beta_epsilon.setter
  def beta_epsilon(self, value):
    self._parms["beta_epsilon"] = value

  @property
  def solver(self):
    return self._parms["solver"]

  @solver.setter
  def solver(self, value):
    self._parms["solver"] = value

  @property
  def standardize(self):
    return self._parms["standardize"]

  @standardize.setter
  def standardize(self, value):
    self._parms["standardize"] = value

  @property
  def family(self):
    return self._parms["family"]

  @family.setter
  def family(self, value):
    self._parms["family"] = value

  @property
  def link(self):
    return self._parms["link"]

  @link.setter
  def link(self, value):
    self._parms["link"] = value

  @property
  def tweedie_variance_power(self):
    return self._parms["tweedie_variance_power"]

  @tweedie_variance_power.setter
  def tweedie_variance_power(self, value):
    self._parms["tweedie_variance_power"] = value

  @property
  def tweedie_link_power(self):
    return self._parms["tweedie_link_power"]

  @tweedie_link_power.setter
  def tweedie_link_power(self, value):
    self._parms["tweedie_link_power"] = value

  @property
  def alpha(self):
    return self._parms["alpha"]

  @alpha.setter
  def alpha(self, value):
    self._parms["alpha"] = value

  @property
  def prior(self):
    return self._parms["prior"]

  @prior.setter
  def prior(self, value):
    self._parms["prior"] = value

  @property
  def lambda_search(self):
    return self._parms["lambda_search"]

  @lambda_search.setter
  def lambda_search(self, value):
    self._parms["lambda_search"] = value

  @property
  def nlambdas(self):
    return self._parms["nlambdas"]

  @nlambdas.setter
  def nlambdas(self, value):
    self._parms["nlambdas"] = value

  @property
  def lambda_min_ratio(self):
    return self._parms["lambda_min_ratio"]

  @lambda_min_ratio.setter
  def lambda_min_ratio(self, value):
    self._parms["lambda_min_ratio"] = value

  @property
  def beta_constraints(self):
    return self._parms["beta_constraints"]

  @beta_constraints.setter
  def beta_constraints(self, value):
    self._parms["beta_constraints"] = value

  @property
  def nfolds(self):
    return self._parms["nfolds"]

  @nfolds.setter
  def nfolds(self, value):
    self._parms["nfolds"] = value

  @property
  def fold_assignment(self):
    return self._parms["fold_assignment"]

  @fold_assignment.setter
  def fold_assignment(self, value):
    self._parms["fold_assignment"] = value

  @property
  def keep_cross_validation_predictions(self):
    return self._parms["keep_cross_validation_predictions"]

  @keep_cross_validation_predictions.setter
  def keep_cross_validation_predictions(self, value):
    self._parms["keep_cross_validation_predictions"] = value

  @property
  def keep_cross_validation_fold_assignment(self):
    return self._parms["keep_cross_validation_fold_assignment"]

  @keep_cross_validation_fold_assignment.setter
  def keep_cross_validation_fold_assignment(self, value):
    self._parms["keep_cross_validation_fold_assignment"] = value

  @property
  def intercept(self):
    return self._parms["intercept"]

  @intercept.setter
  def intercept(self, value):
    self._parms["intercept"] = value

  @property
  def Lambda(self):
    return self._parms["Lambda"]

  @Lambda.setter
  def Lambda(self, value):
    self._parms["Lambda"] = value

  @property
  def max_active_predictors(self):
    return self._parms["max_active_predictors"]

  @max_active_predictors.setter
  def max_active_predictors(self, value):
    self._parms["max_active_predictors"] = value

  @property
  def checkpoint(self):
    return self._parms["checkpoint"]

  @checkpoint.setter
  def checkpoint(self, value):
    self._parms["checkpoint"] = value

  @property
  def objective_epsilon(self):
    return self._parms["objective_epsilon"]

  @objective_epsilon.setter
  def objective_epsilon(self, value):
    self._parms["objective_epsilon"] = value

  @property
  def gradient_epsilon(self):
    return self._parms["gradient_epsilon"]

  @gradient_epsilon.setter
  def gradient_epsilon(self, value):
    self._parms["gradient_epsilon"] = value

  @property
  def non_negative(self):
    return self._parms["non_negative"]

  @non_negative.setter
  def non_negative(self, value):
    self._parms["non_negative"] = value

  @property
  def compute_p_values(self):
    return self._parms["compute_p_values"]

  @compute_p_values.setter
  def compute_p_values(self, value):
    self._parms["compute_p_values"] = value

  @property
  def remove_collinear_columns(self):
    return self._parms["remove_collinear_columns"]

  @remove_collinear_columns.setter
  def remove_collinear_columns(self, value):
    self._parms["remove_collinear_columns"] = value

  @property
  def missing_values_handling(self):
    return self._parms["missing_values_handling"]

  @missing_values_handling.setter
  def missing_values_handling(self, value):
    self._parms["missing_values_handling"] = value

  """
  Extract full regularization path explored during lambda search from glm model.

  Parameters:

  model - source lambda search model

  """
  @staticmethod
  def getGLMRegularizationPath(model):
    x = H2OConnection.get_json("GetGLMRegPath",model=model._model_json['model_id']['name'])
    ns = x.pop('coefficient_names')
    res = {'lambdas':x['lambdas'],'explained_deviance_train':x['explained_deviance_train'],'explained_deviance_valid':x['explained_deviance_valid']}
    res['coefficients'] = [dict(zip(ns,y)) for y in x['coefficients']]
    if 'coefficients_std' in x:
      res['coefficients_std'] = [dict(zip(ns,y)) for y in x['coefficients_std']]
    return res

  """
  Create a custom GLM model using the given coefficients.
  Needs to be passed source model trained on the dataset to extract the dataset information from.

  Parameters:
    model - source model, used for extracting dataset information
    coefs - dictionary containing model coefficients
    threshold - (optional, only for binomial) decision threshold used for classification

  """
  @staticmethod
  def makeGLMModel(model, coefs, threshold=.5):
    model_json = H2OConnection.post_json("MakeGLMModel",model=model._model_json['model_id']['name'], names=coefs.keys(), beta = coefs.values(), threshold = threshold)
    m = H2OGeneralizedLinearEstimator()
    m._resolve_model(model_json['model_id']['name'], model_json)
    return m

<commit_msg>Updated makeGLMModel python call to be compatible with python3<commit_after>from .estimator_base import H2OEstimator
from h2o.connection import H2OConnection

class H2OGeneralizedLinearEstimator(H2OEstimator):
  """Build a Generalized Linear Model
    Fit a generalized linear model, specified by a response variable, a set of predictors,
    and a description of the error distribution.

    Parameters
    ----------
      model_id : str, optional
        The unique id assigned to the resulting model. If none is given, an id will
        automatically be generated.

      max_iterations : int
        A non-negative integer specifying the maximum number of iterations.

      beta_epsilon : int
        A non-negative number specifying the magnitude of the maximum difference between
        the coefficient estimates from successive iterations. Defines the convergence
        criterion.

      solver : str
        A character string specifying the solver used: IRLSM (supports more features),
        L_BFGS (scales better for datasets with many columns)

      standardize : bool
        Indicates whether the numeric predictors should be standardized to have a mean of
        0 and a variance of 1 prior to training the models.

      family : str
        A character string specifying the distribution of the model:
        gaussian, binomial, multinomial, poisson, gamma, tweedie.

      link : str
        A character string specifying the link function. The default is the canonical
        link for the family. The supported links for each of the family specifications are
        "gaussian" - "identity", "log", "inverse"
        "binomial" - "logit", "log"
        "multinomial" - "multinomial"
        "poisson" - "log", "identity"
        "gamma" - "inverse", "log", "identity"
        "tweedie" - "tweedie"

      tweedie_variance_power : int
        numeric specifying the power for the variance function when family = "tweedie".

      tweedie_link_power : int
        A numeric specifying the power for the link function when family = "tweedie".

      alpha : float
        A numeric in [0, 1] specifying the elastic-net mixing parameter.
        The elastic-net penalty is defined to be
        eqn{P(\alpha,\beta) = (1-\alpha)/2||\beta||_2^2 +
        \alpha||\beta||_1 = \sum_j [(1-\alpha)/2 \beta_j^2 + \alpha|\beta_j|],
        making alpha = 1 the lasso penalty and alpha = 0 the ridge penalty.

      Lambda : float
        A non-negative shrinkage parameter for the elastic-net, which multiplies
        \eqn{P(\alpha,\beta) in the objective function.
        When Lambda = 0, no elastic-net penalty is applied and ordinary generalized linear
        models are fit.

      prior : float, optional
        A numeric specifying the prior probability of class 1 in the response when
        family = "binomial". The default prior is the observational frequency of class 1.
        Must be from (0,1) exclusive range or None (no prior).

      lambda_search : bool
        A logical value indicating whether to conduct a search over the space of lambda
        values starting from the lambda max, given lambda is interpreted as lambda minself.

      nlambdas : int
        The number of lambda values to use when lambda_search = TRUE.

      lambda_min_ratio : float
        Smallest value for lambda as a fraction of lambda.max. By default if the number of
        observations is greater than the the number of variables then
        lambda_min_ratio = 0.0001; if the number of observations is less than the number
        of variables then lambda_min_ratio = 0.01.

      beta_constraints : H2OFrame
        A data.frame or H2OParsedData object with the columns
        ["names", "lower_bounds", "upper_bounds", "beta_given"],
        where each row corresponds to a predictor in the GLM.
        "names" contains the predictor names, "lower"/"upper_bounds",
        are the lower and upper bounds of beta, and "beta_given" is some supplied starting
        values.

      nfolds : int, optional
        Number of folds for cross-validation. If nfolds >= 2, then validation must
        remain empty.

      fold_assignment : str
        Cross-validation fold assignment scheme, if fold_column is not specified Must be
        "AUTO", "Random" or "Modulo"

      keep_cross_validation_predictions : bool
        Whether to keep the predictions of the cross-validation models

      keep_cross_validation_fold_assignment : bool
        Whether to keep the cross-validation fold assignment.

      intercept : bool
        Logical, include constant term (intercept) in the model

      max_active_predictors : int, optional
        Convergence criteria for number of predictors when using L1 penalty.

      missing_values_handling : str
        A character string specifying how to handle missing value:
        "MeanImputation","Skip".

      interactions : list, optional
        A list of column names to interact. All pairwise combinations of columns will be
        interacted.
      max_runtime_secs: int, optional
        Maximum allowed runtime, model will stop running after reaching the limit and return whatever result it has at the moment.

    Returns
    -------
      A subclass of ModelBase is returned. The specific subclass depends on the machine
      learning task at hand (if it's binomial classification, then an H2OBinomialModel
      is returned, if it's regression then a H2ORegressionModel is returned). The default
      print-out of the models is shown, but further GLM-specifc information can be
      queried out of the object. Upon completion of the GLM, the resulting object has
      coefficients, normalized coefficients, residual/null deviance, aic, and a host of
      model metrics including MSE, AUC (for logistic regression), degrees of freedom, and
      confusion matrices.
    """
  def __init__(self, model_id=None, max_iterations=None, beta_epsilon=None, solver=None,
               standardize=None, family=None, link=None, tweedie_variance_power=None,
               tweedie_link_power=None, alpha=None, prior=None, lambda_search=None,
               nlambdas=None, lambda_min_ratio=None, beta_constraints=None, nfolds=None,
               fold_assignment=None, keep_cross_validation_predictions=None,
               keep_cross_validation_fold_assignment=None,
               intercept=None, Lambda=None, max_active_predictors=None, checkpoint=None,
               objective_epsilon=None, gradient_epsilon=None, non_negative=False,
               compute_p_values=False, remove_collinear_columns=False,
               missing_values_handling=None, interactions=None):
    super(H2OGeneralizedLinearEstimator, self).__init__()
    self._parms = locals()
    self._parms = {k: v for k, v in self._parms.items() if k != "self"}
    self._parms["lambda"] = self._parms.pop("Lambda")

  @property
  def max_iterations(self):
    return self._parms["max_iterations"]

  @max_iterations.setter
  def max_iterations(self, value):
    self._parms["max_iterations"] = value

  @property
  def beta_epsilon(self):
    return self._parms["beta_epsilon"]

  @beta_epsilon.setter
  def beta_epsilon(self, value):
    self._parms["beta_epsilon"] = value

  @property
  def solver(self):
    return self._parms["solver"]

  @solver.setter
  def solver(self, value):
    self._parms["solver"] = value

  @property
  def standardize(self):
    return self._parms["standardize"]

  @standardize.setter
  def standardize(self, value):
    self._parms["standardize"] = value

  @property
  def family(self):
    return self._parms["family"]

  @family.setter
  def family(self, value):
    self._parms["family"] = value

  @property
  def link(self):
    return self._parms["link"]

  @link.setter
  def link(self, value):
    self._parms["link"] = value

  @property
  def tweedie_variance_power(self):
    return self._parms["tweedie_variance_power"]

  @tweedie_variance_power.setter
  def tweedie_variance_power(self, value):
    self._parms["tweedie_variance_power"] = value

  @property
  def tweedie_link_power(self):
    return self._parms["tweedie_link_power"]

  @tweedie_link_power.setter
  def tweedie_link_power(self, value):
    self._parms["tweedie_link_power"] = value

  @property
  def alpha(self):
    return self._parms["alpha"]

  @alpha.setter
  def alpha(self, value):
    self._parms["alpha"] = value

  @property
  def prior(self):
    return self._parms["prior"]

  @prior.setter
  def prior(self, value):
    self._parms["prior"] = value

  @property
  def lambda_search(self):
    return self._parms["lambda_search"]

  @lambda_search.setter
  def lambda_search(self, value):
    self._parms["lambda_search"] = value

  @property
  def nlambdas(self):
    return self._parms["nlambdas"]

  @nlambdas.setter
  def nlambdas(self, value):
    self._parms["nlambdas"] = value

  @property
  def lambda_min_ratio(self):
    return self._parms["lambda_min_ratio"]

  @lambda_min_ratio.setter
  def lambda_min_ratio(self, value):
    self._parms["lambda_min_ratio"] = value

  @property
  def beta_constraints(self):
    return self._parms["beta_constraints"]

  @beta_constraints.setter
  def beta_constraints(self, value):
    self._parms["beta_constraints"] = value

  @property
  def nfolds(self):
    return self._parms["nfolds"]

  @nfolds.setter
  def nfolds(self, value):
    self._parms["nfolds"] = value

  @property
  def fold_assignment(self):
    return self._parms["fold_assignment"]

  @fold_assignment.setter
  def fold_assignment(self, value):
    self._parms["fold_assignment"] = value

  @property
  def keep_cross_validation_predictions(self):
    return self._parms["keep_cross_validation_predictions"]

  @keep_cross_validation_predictions.setter
  def keep_cross_validation_predictions(self, value):
    self._parms["keep_cross_validation_predictions"] = value

  @property
  def keep_cross_validation_fold_assignment(self):
    return self._parms["keep_cross_validation_fold_assignment"]

  @keep_cross_validation_fold_assignment.setter
  def keep_cross_validation_fold_assignment(self, value):
    self._parms["keep_cross_validation_fold_assignment"] = value

  @property
  def intercept(self):
    return self._parms["intercept"]

  @intercept.setter
  def intercept(self, value):
    self._parms["intercept"] = value

  @property
  def Lambda(self):
    return self._parms["Lambda"]

  @Lambda.setter
  def Lambda(self, value):
    self._parms["Lambda"] = value

  @property
  def max_active_predictors(self):
    return self._parms["max_active_predictors"]

  @max_active_predictors.setter
  def max_active_predictors(self, value):
    self._parms["max_active_predictors"] = value

  @property
  def checkpoint(self):
    return self._parms["checkpoint"]

  @checkpoint.setter
  def checkpoint(self, value):
    self._parms["checkpoint"] = value

  @property
  def objective_epsilon(self):
    return self._parms["objective_epsilon"]

  @objective_epsilon.setter
  def objective_epsilon(self, value):
    self._parms["objective_epsilon"] = value

  @property
  def gradient_epsilon(self):
    return self._parms["gradient_epsilon"]

  @gradient_epsilon.setter
  def gradient_epsilon(self, value):
    self._parms["gradient_epsilon"] = value

  @property
  def non_negative(self):
    return self._parms["non_negative"]

  @non_negative.setter
  def non_negative(self, value):
    self._parms["non_negative"] = value

  @property
  def compute_p_values(self):
    return self._parms["compute_p_values"]

  @compute_p_values.setter
  def compute_p_values(self, value):
    self._parms["compute_p_values"] = value

  @property
  def remove_collinear_columns(self):
    return self._parms["remove_collinear_columns"]

  @remove_collinear_columns.setter
  def remove_collinear_columns(self, value):
    self._parms["remove_collinear_columns"] = value

  @property
  def missing_values_handling(self):
    return self._parms["missing_values_handling"]

  @missing_values_handling.setter
  def missing_values_handling(self, value):
    self._parms["missing_values_handling"] = value

  """
  Extract full regularization path explored during lambda search from glm model.

  Parameters:

  model - source lambda search model

  """
  @staticmethod
  def getGLMRegularizationPath(model):
    x = H2OConnection.get_json("GetGLMRegPath",model=model._model_json['model_id']['name'])
    ns = x.pop('coefficient_names')
    res = {'lambdas':x['lambdas'],'explained_deviance_train':x['explained_deviance_train'],'explained_deviance_valid':x['explained_deviance_valid']}
    res['coefficients'] = [dict(zip(ns,y)) for y in x['coefficients']]
    if 'coefficients_std' in x:
      res['coefficients_std'] = [dict(zip(ns,y)) for y in x['coefficients_std']]
    return res

  """
  Create a custom GLM model using the given coefficients.
  Needs to be passed source model trained on the dataset to extract the dataset information from.

  Parameters:
    model - source model, used for extracting dataset information
    coefs - dictionary containing model coefficients
    threshold - (optional, only for binomial) decision threshold used for classification

  """
  @staticmethod
  def makeGLMModel(model, coefs, threshold=.5):
    model_json = H2OConnection.post_json("MakeGLMModel",model=model._model_json['model_id']['name'], names=list(coefs.keys()), beta = list(coefs.values()), threshold = threshold)
    m = H2OGeneralizedLinearEstimator()
    m._resolve_model(model_json['model_id']['name'], model_json)
    return m

<|endoftext|>


<commit_before>import bcrypt

from flask import flash, get_flashed_messages, redirect, render_template, \
	request, url_for, abort
from flask import Blueprint, Markup
from flask.ext.login import current_user, login_user, logout_user

from viaduct import application, db, login_manager
from viaduct.helpers import flash_form_errors
from forms import SignUpForm, SignInForm
from models import User, UserPermission

blueprint = Blueprint('user', __name__)

@login_manager.user_loader
def load_user(id):
	# The hook used by the login manager to get the user from the database by
	# user ID.
	return User.query.get(int(id))

def load_anonymous_user():
	return User.query.filter(User.email=='anonymous').first()

@blueprint.route('/sign-up/', methods=['GET', 'POST'])
def sign_up():
	# Redirect the user to the index page if he or she has been authenticated
	# already.
	if current_user and current_user.is_authenticated():
		return redirect(url_for('page.get_page'))

	form = SignUpForm(request.form)

	if form.validate_on_submit():
		user = User(form.email.data, bcrypt.hashpw(form.password.data, bcrypt.gensalt()),
			form.first_name.data, form.last_name.data)

		db.session.add(user)
		db.session.commit()

		flash('You\'ve signed up successfully.')

		login_user(user)

		return redirect(url_for('page.get_page'))
	else:
		flash_form_errors(form)

	return render_template('user/sign_up.htm', form=form)

@blueprint.route('/sign-in/', methods=['GET', 'POST'])
def sign_in():
	# Redirect the user to the index page if he or she has been authenticated
	# already.
	if current_user and current_user.is_authenticated():
		return redirect(url_for('page.get_page'))

	form = SignInForm(request.form)

	if form.validate_on_submit():
		valid_form = True

		user = User.query.filter(User.email==form.email.data).first()

		# Check if the user does exist, and if the passwords do match.
		if not user or bcrypt.hashpw(form.password.data, user.password) != user.password:
			flash('The credentials that have been specified are invalid.', 'error')
			valid_form = False

		if valid_form:
			# Notify the login manager that the user has been signed in.
			login_user(user)

			flash('You\'ve been signed in successfully.')

			# return redirect(url_for('page.get_page'))
			return redirect('/')
	else:
		flash_form_errors(form)

	return render_template('user/sign_in.htm', form=form)

@blueprint.route('/sign-out/')
def sign_out():
	# Notify the login manager that the user has been signed out.
	logout_user()

	flash('You\'ve been signed out.')

	# FIX THIS!
	# return redirect(url_for('page.get_page'))
	return redirect('/')

@blueprint.route('/users/', methods=['GET', 'POST'])
@blueprint.route('/users/<int:page>/', methods=['GET', 'POST'])
def view(page=1):
	if not UserPermission.get_user_rights(current_user)['view']:
		abort(403)

	# persumably, if the method is a post we have selected stuff to delete,
	# similary to groups
	if request.method == 'POST':
		user_ids = request.form.getlist('select')

		users = User.query.filter(User.id.in_(user_ids)).all()

		for user in users:
			db.session.delete(user)

		db.session.commit()

		if len(users) > 1:
			flash('The selected users have been deleted.', 'success')
		else:
			flash('The selected user has been deleted.', 'success')

		redirect(url_for('user.view'))

	# Get a list of users to render for the current page.
	users = User.query.paginate(page, 15, False)

	return render_template('user/view.htm', users=users)

<commit_msg>Quick fix.<commit_after>import bcrypt

from flask import flash, get_flashed_messages, redirect, render_template, \
	request, url_for, abort
from flask import Blueprint, Markup
from flask.ext.login import current_user, login_user, logout_user

from viaduct import application, db, login_manager
from viaduct.helpers import flash_form_errors
from forms import SignUpForm, SignInForm
from models import User, UserPermission

blueprint = Blueprint('user', __name__)

@login_manager.user_loader
def load_user(id):
	# The hook used by the login manager to get the user from the database by
	# user ID.
	return User.query.get(int(id))

def load_anonymous_user():
	return User.query.filter(User.email=='anonymous').first()

@blueprint.route('/sign-up/', methods=['GET', 'POST'])
def sign_up():
	# Redirect the user to the index page if he or she has been authenticated
	# already.
	if current_user and current_user.is_authenticated():
		return redirect(url_for('page.get_page'))

	form = SignUpForm(request.form)

	if form.validate_on_submit():
		user = User(form.email.data, bcrypt.hashpw(form.password.data, bcrypt.gensalt()),
			form.first_name.data, form.last_name.data)

		db.session.add(user)
		db.session.commit()

		flash('You\'ve signed up successfully.')

		login_user(user)

		return redirect(url_for('page.get_page'))
	else:
		flash_form_errors(form)

	return render_template('user/sign_up.htm', form=form)

@blueprint.route('/sign-in/', methods=['GET', 'POST'])
def sign_in():
	# Redirect the user to the index page if he or she has been authenticated
	# already.
	if current_user and current_user.is_authenticated():
		return redirect(url_for('page2.get_page'))

	form = SignInForm(request.form)

	if form.validate_on_submit():
		valid_form = True

		user = User.query.filter(User.email==form.email.data).first()

		# Check if the user does exist, and if the passwords do match.
		if not user or bcrypt.hashpw(form.password.data, user.password) != user.password:
			flash('The credentials that have been specified are invalid.', 'error')
			valid_form = False

		if valid_form:
			# Notify the login manager that the user has been signed in.
			login_user(user)

			flash('You\'ve been signed in successfully.')

			# return redirect(url_for('page.get_page'))
			return redirect('/')
	else:
		flash_form_errors(form)

	return render_template('user/sign_in.htm', form=form)

@blueprint.route('/sign-out/')
def sign_out():
	# Notify the login manager that the user has been signed out.
	logout_user()

	flash('You\'ve been signed out.')

	# FIX THIS!
	# return redirect(url_for('page.get_page'))
	return redirect('/')

@blueprint.route('/users/', methods=['GET', 'POST'])
@blueprint.route('/users/<int:page>/', methods=['GET', 'POST'])
def view(page=1):
	if not UserPermission.get_user_rights(current_user)['view']:
		abort(403)

	# persumably, if the method is a post we have selected stuff to delete,
	# similary to groups
	if request.method == 'POST':
		user_ids = request.form.getlist('select')

		users = User.query.filter(User.id.in_(user_ids)).all()

		for user in users:
			db.session.delete(user)

		db.session.commit()

		if len(users) > 1:
			flash('The selected users have been deleted.', 'success')
		else:
			flash('The selected user has been deleted.', 'success')

		redirect(url_for('user.view'))

	# Get a list of users to render for the current page.
	users = User.query.paginate(page, 15, False)

	return render_template('user/view.htm', users=users)

<|endoftext|>


<commit_before>def calculateTip(total, rate):
    return total * rate

print "The tip is "
print calculateTip(37.50, .15)
<commit_msg>update tip-calculator to take user input<commit_after>
#takes in user input and prints out the tip for the total bsaed on the rate
def calculateTip():
    total = float(input("Please enter the total amount as a float: "))
    rate  = float(input("Please enter the tax rate: "))
    tip = total * rate
    print "The tip is: ", tip
    return tip

calculateTip()
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# md2epub.py
#
# by Ben Crowder
# http://bencrowder.net/
#
# Based off Matt Turner's GetBook.py
# http://staff.washington.edu/mdturner/personal.htm
#

import urllib, re, os, zipfile, glob, shutil, sys, markdown2, string, datetime

class Chapter:
	title = ''
	filename = ''
	htmlfile = ''
	id = ''

class EPub:
	title = ''
	author = ''
	lang = 'en-US'
	basename = ''			# base name of ePub (before the extension)
	path = ''				# directory ePub will be stored in
	bookid = ''				# unique identifier
	url = ''				# URL for this book
	css = ''				# CSS file
	cover = ''				# book cover art
	images = []				# list of images to be included
	content = []			# list of Chapters

	def cleanup(self):
		os.rmdir(self.path)

	def save(self):
		# get current working directory
		cwd = os.getcwd()

		# create directory for the ePub
		os.mkdir(self.path)
		os.chdir(self.path)

		try:
			# make mimetype
			f = open('mimetype','w')
			f.write('application/epub+zip')
			f.close()

			# make the META-INF/container.xml
			os.mkdir('META-INF')
			os.chdir('META-INF')
			f = open('container.xml','w')
			f.write('''<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
	<rootfiles>
		<rootfile full-path="OEBPS/metadata.opf" media-type="application/oebps-package+xml" />
	</rootfiles>
</container>''')
			f.close()
			os.chdir('..')

			# make OEBPS/metadata.opf
			os.mkdir('OEBPS')
			os.chdir('OEBPS')
			f = open('metadata.opf','w')
			f.write('''<?xml version="1.0"?>
<package version="2.0" xmlns="http://www.idpf.org/2007/opf"
		 unique-identifier="bookid">
	<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
		<dc:title>''' + self.title + '''</dc:title>
		<dc:creator opf:role="aut">''' + self.author + '''</dc:creator>
		<dc:language>''' + self.lang + '''</dc:language>
		<dc:identifier id="bookid">''' + self.url + '''</dc:identifier>''')
			if self.cover:
				f.write('\t\t<meta name="cover" content="book-cover" />')

			f.write('''
	</metadata>
	<manifest>
		<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml" />
''')

			if self.css:
				f.write('\t\t<item id="style" href="' + os.path.basename(self.css) + '" media-type="application/css" />')

			if self.cover:
				imagefile = os.path.basename(self.cover)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('\t\t<item id="book-cover" href="' + imagefile + '" media-type="image/' + ext + '" />')

			for chapter in self.content:
				f.write('''
		<item id="''' + chapter.id + '''" href="''' + chapter.htmlfile + '''" media-type="application/xhtml+xml" />''')

			for image in self.images:
				imagefile = os.path.basename(image)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('''
		<item id="''' + imagefile + '''" href="''' + imagefile + '''" media-type="image/''' + ext + '''" />''')

			f.write('''
	</manifest>
	<spine toc="ncx">''')

			for chapter in self.content:
				f.write('''
		<itemref idref="''' + chapter.id + '''"/>''')

			f.write('''
	</spine>
</package>''')

			f.close()

			# make toc.ncx
			f = open('toc.ncx','w')
			f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
	<head>
		<meta name="dtb:uid" content="''' + self.url + '''" />
		<meta name="dtb:depth" content="1" />
		<meta name="dtb:totalPageCount" content="0" />
		<meta name="dtb:maxPageNumber" content="0" />
	</head>
	<docTitle>
		<text>''' + self.title + '''</text>
	</docTitle>
	<navMap>''')

			n = 1
			for chapter in self.content:
				f.write('''
		<navPoint id="''' + chapter.id + '''" playOrder="''' + str(n) + '''">
			<navLabel><text>''' + chapter.title + '''</text></navLabel>
			<content src="''' + chapter.htmlfile + '''" />
		</navPoint>''')
				n += 1

			f.write('''
	</navMap>
</ncx>''')

			f.close()

			# convert the texts to Markdown and save in the directory
			for chapter in self.content:
				input = open('../../' + chapter.filename, 'r')
				f = open(chapter.htmlfile, 'w')
				sourcetext = input.read()
				input.close()

				# write HTML header
				f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">

<html xmlns="http://www.w3.org/1999/xhtml">
<head>
	<title>''' + self.title + '''</title>''')
				if self.css:
					f.write('\n\t<link rel="stylesheet" type="text/css" href="' + os.path.basename(self.css) + '" />')
				f.write('''
	<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
</head>
<body>''')

				# write the Markdowned text
				f.write(markdown2.markdown(sourcetext).encode('utf-8'))

				# write HTML footer
				f.write('''
</body>
</html>''')

				f.close()

			# if there's a CSS file, copy it in
			if self.css:
				css = open('../../' + self.css, 'r')
				csstext = css.read()
				css.close()
				f = open(os.path.basename(self.css), 'w')
				f.write(csstext)
				f.close()

			# copy cover art into the directory
			if self.cover:
				dest = os.path.basename(self.cover)
				shutil.copyfile('../../' + self.cover, dest)

			# copy images into the directory
			for image in self.images:
				dest = os.path.basename(image)
				shutil.copyfile('../../' + image, dest)

			# now zip the ePub up
			os.chdir('../..')
			file = zipfile.ZipFile(self.basename + '.epub', "w")
			os.chdir(self.path)

			file.write('mimetype','mimetype', zipfile.ZIP_STORED)
			file.write('META-INF/container.xml', 'META-INF/container.xml', zipfile.ZIP_DEFLATED)
			for filename in glob.glob("OEBPS/*"):
				file.write(filename, filename, zipfile.ZIP_DEFLATED)

			# and remove the directory
			os.chdir('..')
			shutil.rmtree(self.path)
		except:
			# if something went wrong, remove the temp directory
			os.chdir(cwd)
			shutil.rmtree(self.path)

def process_book(filename):
	epub = EPub()

	now = datetime.datetime.now()
	epub.basename = filename.split('.')[0]
	epub.path = '%s_%s-%s-%s_%s-%s-%s' % (epub.basename, now.year, now.month, now.day, now.hour, now.minute, now.second)
	epub.url = 'http://localhost/' + epub.path		# dummy URL, replaced in control file

	fh = open(filename, 'r')
	for line in fh.readlines():
		if ':' in line:					# keywords
			values = line.split(':')
			keyword = values[0].strip()
			value = values[1].strip()
			if keyword == 'Title':
				epub.title = value
			elif keyword == 'Author':
				epub.author = value
			elif keyword == 'Language':
				epub.lang = value
			elif keyword == 'URL':
				epub.url = ''.join([values[1].strip(), ':', values[2].strip()])
			elif keyword == 'Image' or keyword == 'Images':
				images = value.split(',')
				# split list
				for image in images:
					epub.images.append(image.strip())
			elif keyword == 'CSS':
				epub.css = value
			elif keyword == 'Cover':
				epub.cover = value
		elif '|' in line:				# contents
			values = line.split('|')
			title = values[0].strip()
			filename = values[1].strip()

			chapter = Chapter()
			chapter.title = values[0].strip()
			chapter.filename = values[1].strip()

			# replace extension with .html
			basename = os.path.basename(chapter.filename)
			chapter.htmlfile = os.path.splitext(basename)[0] + '.html'

			# for the ID, lowercase it all, strip punctuation, and replace spaces with underscores
			chapter.id = re.sub(r'[^a-zA-Z0-9]', r'', chapter.title.lower()).replace(' ', '_')
		 	# chapter.id = chapter.id.translate(string.maketrans('',''), string.punctuation).replace(' ', '_')

			epub.content.append(chapter)
		else:
			if line[0] == '#' or not line.strip():
				# ignore comments and blank lines
				pass
			else:
				print 'Error!'

	fh.close()

	# create a (hopefully unique) book ID
	epub.bookid = '[' + epub.title + '|' + epub.author + ']'

	return epub

##### Main

def main():
	if (len(sys.argv) > 1):
		epub = process_book(sys.argv[1])
		epub.save()

if __name__ == "__main__":
	main()
<commit_msg>Added parse support for two-level TOCs<commit_after>#!/usr/bin/python
#
# md2epub.py
#
# by Ben Crowder
# http://bencrowder.net/
#
# Based off Matt Turner's GetBook.py
# http://staff.washington.edu/mdturner/personal.htm
#

import urllib, re, os, zipfile, glob, shutil, sys, markdown2, string, datetime

class Chapter:
	title = ''
	filename = ''
	htmlfile = ''
	id = ''
	children = []

class EPub:
	title = ''
	author = ''
	lang = 'en-US'
	basename = ''			# base name of ePub (before the extension)
	path = ''				# directory ePub will be stored in
	bookid = ''				# unique identifier
	url = ''				# URL for this book
	css = ''				# CSS file
	cover = ''				# book cover art
	images = []				# list of images to be included
	content = []			# list of Chapters

	def cleanup(self):
		os.rmdir(self.path)

	def save(self):
		# get current working directory
		cwd = os.getcwd()

		# create directory for the ePub
		os.mkdir(self.path)
		os.chdir(self.path)

		try:
			# make mimetype
			f = open('mimetype','w')
			f.write('application/epub+zip')
			f.close()

			# make the META-INF/container.xml
			os.mkdir('META-INF')
			os.chdir('META-INF')
			f = open('container.xml','w')
			f.write('''<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
	<rootfiles>
		<rootfile full-path="OEBPS/metadata.opf" media-type="application/oebps-package+xml" />
	</rootfiles>
</container>''')
			f.close()
			os.chdir('..')

			# make OEBPS/metadata.opf
			os.mkdir('OEBPS')
			os.chdir('OEBPS')
			f = open('metadata.opf','w')
			f.write('''<?xml version="1.0"?>
<package version="2.0" xmlns="http://www.idpf.org/2007/opf"
		 unique-identifier="bookid">
	<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
		<dc:title>''' + self.title + '''</dc:title>
		<dc:creator opf:role="aut">''' + self.author + '''</dc:creator>
		<dc:language>''' + self.lang + '''</dc:language>
		<dc:identifier id="bookid">''' + self.url + '''</dc:identifier>''')
			if self.cover:
				f.write('\t\t<meta name="cover" content="book-cover" />')

			f.write('''
	</metadata>
	<manifest>
		<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml" />
''')

			if self.css:
				f.write('\t\t<item id="style" href="' + os.path.basename(self.css) + '" media-type="application/css" />')

			if self.cover:
				imagefile = os.path.basename(self.cover)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('\t\t<item id="book-cover" href="' + imagefile + '" media-type="image/' + ext + '" />')

			for chapter in self.content:
				f.write('''
		<item id="''' + chapter.id + '''" href="''' + chapter.htmlfile + '''" media-type="application/xhtml+xml" />''')

			for image in self.images:
				imagefile = os.path.basename(image)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('''
		<item id="''' + imagefile + '''" href="''' + imagefile + '''" media-type="image/''' + ext + '''" />''')

			f.write('''
	</manifest>
	<spine toc="ncx">''')

			for chapter in self.content:
				f.write('''
		<itemref idref="''' + chapter.id + '''"/>''')

			f.write('''
	</spine>
</package>''')

			f.close()

			# make toc.ncx
			f = open('toc.ncx','w')
			f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
	<head>
		<meta name="dtb:uid" content="''' + self.url + '''" />
		<meta name="dtb:depth" content="1" />
		<meta name="dtb:totalPageCount" content="0" />
		<meta name="dtb:maxPageNumber" content="0" />
	</head>
	<docTitle>
		<text>''' + self.title + '''</text>
	</docTitle>
	<navMap>''')

			n = 1
			for chapter in self.content:
				f.write('''
		<navPoint id="''' + chapter.id + '''" playOrder="''' + str(n) + '''">
			<navLabel><text>''' + chapter.title + '''</text></navLabel>
			<content src="''' + chapter.htmlfile + '''" />
		</navPoint>''')
				n += 1

			f.write('''
	</navMap>
</ncx>''')

			f.close()

			# convert the texts to Markdown and save in the directory
			for chapter in self.content:
				input = open('../../' + chapter.filename, 'r')
				f = open(chapter.htmlfile, 'w')
				sourcetext = input.read()
				input.close()

				# write HTML header
				f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">

<html xmlns="http://www.w3.org/1999/xhtml">
<head>
	<title>''' + self.title + '''</title>''')
				if self.css:
					f.write('\n\t<link rel="stylesheet" type="text/css" href="' + os.path.basename(self.css) + '" />')
				f.write('''
	<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
</head>
<body>''')

				# write the Markdowned text
				f.write(markdown2.markdown(sourcetext).encode('utf-8'))

				# write HTML footer
				f.write('''
</body>
</html>''')

				f.close()

			# if there's a CSS file, copy it in
			if self.css:
				css = open('../../' + self.css, 'r')
				csstext = css.read()
				css.close()
				f = open(os.path.basename(self.css), 'w')
				f.write(csstext)
				f.close()

			# copy cover art into the directory
			if self.cover:
				dest = os.path.basename(self.cover)
				shutil.copyfile('../../' + self.cover, dest)

			# copy images into the directory
			for image in self.images:
				dest = os.path.basename(image)
				shutil.copyfile('../../' + image, dest)

			# now zip the ePub up
			os.chdir('../..')
			file = zipfile.ZipFile(self.basename + '.epub', "w")
			os.chdir(self.path)

			file.write('mimetype','mimetype', zipfile.ZIP_STORED)
			file.write('META-INF/container.xml', 'META-INF/container.xml', zipfile.ZIP_DEFLATED)
			for filename in glob.glob("OEBPS/*"):
				file.write(filename, filename, zipfile.ZIP_DEFLATED)

			# and remove the directory
			os.chdir('..')
			shutil.rmtree(self.path)
		except:
			# if something went wrong, remove the temp directory
			os.chdir(cwd)
			shutil.rmtree(self.path)

def process_book(filename):
	epub = EPub()

	now = datetime.datetime.now()
	epub.basename = filename.split('.')[0]
	epub.path = '%s_%s-%s-%s_%s-%s-%s' % (epub.basename, now.year, now.month, now.day, now.hour, now.minute, now.second)
	epub.url = 'http://localhost/' + epub.path		# dummy URL, replaced in control file

	fh = open(filename, 'r')
	for line in fh.readlines():
		if ':' in line:					# keywords
			values = line.split(':')
			keyword = values[0].strip()
			value = values[1].strip()
			if keyword == 'Title':
				epub.title = value
			elif keyword == 'Author':
				epub.author = value
			elif keyword == 'Language':
				epub.lang = value
			elif keyword == 'URL':
				epub.url = ''.join([values[1].strip(), ':', values[2].strip()])
			elif keyword == 'Image' or keyword == 'Images':
				images = value.split(',')
				# split list
				for image in images:
					epub.images.append(image.strip())
			elif keyword == 'CSS':
				epub.css = value
			elif keyword == 'Cover':
				epub.cover = value
		elif '|' in line:				# contents
			values = line.split('|')
			title = values[0].strip()
			filename = values[1].strip()

			chapter = Chapter()
			chapter.title = values[0].strip()
			chapter.filename = values[1].strip()

			# replace extension with .html
			basename = os.path.basename(chapter.filename)
			chapter.htmlfile = os.path.splitext(basename)[0] + '.html'

			# for the ID, lowercase it all, strip punctuation, and replace spaces with underscores
			chapter.id = re.sub(r'[^a-zA-Z0-9]', r'', chapter.title.lower()).replace(' ', '_')

			# if this is a child chapter, add to parent; otherwise add to root
			if line[0] == "\t":
				if lastchapter: # make sure we have something to add to
					print 'Appending %s to %s' % (chapter.title, lastchapter.title)
					lastchapter.children.append(chapter)
				else:
					print 'Appending %s to root (oops)' % (chapter.title)
					epub.content.append(chapter)			# plan B
			else:
				print 'Appending %s to root' % (chapter.title)
				epub.content.append(chapter)

				# for hierarchies, keep a reference to the last chapter
				lastchapter = chapter
		else:
			if line[0] == '#' or not line.strip():
				# ignore comments and blank lines
				pass
			else:
				print 'Error!'

	fh.close()

	# create a (hopefully unique) book ID
	epub.bookid = '[' + epub.title + '|' + epub.author + ']'

	return epub

##### Main

def main():
	if (len(sys.argv) > 1):
		epub = process_book(sys.argv[1])
		epub.save()

if __name__ == "__main__":
	main()
<|endoftext|>