<commit_before>#!/usr/bin/python

import os
import json
import urllib
import sys
import collections
import dateutil.parser
import subprocess
import httplib
import sqlite3
import datetime
import scraperwiki
import httplib
import random
import datetime

from secrets import *

MAX_TO_GET=100000

# Horrendous hack to work around some Twitter / Python incompatibility
# http://bobrochel.blogspot.co.nz/2010/11/bad-servers-chunked-encoding-and.html
def patch_http_response_read(func):
    def inner(*args):
        try:
            return func(*args)
        except httplib.IncompleteRead, e:
            return e.partial

    return inner
httplib.HTTPResponse.read = patch_http_response_read(httplib.HTTPResponse.read)

# Make sure you install this version of "twitter":
# http://pypi.python.org/pypi/twitter
# http://mike.verdone.ca/twitter/
# https://github.com/sixohsix/twitter
import twitter

#########################################################################
# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which
# currently only has a function for PIN authentication, not redirect)
from twitter.api import Twitter
from twitter.oauth import OAuth, write_token_file, read_token_file
from twitter.oauth_dance import parse_oauth_tokens
def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):
    # Verification happens in two stages...

    # 1) If we haven't done a pre-verification yet... Then we get credentials from Twitter
    # that will be used to sign our redirect to them, find the redirect, and instruct the Javascript
    # that called us to do the redirect.
    if not os.path.exists(CREDS_PRE_VERIFIY):
        twitter = Twitter(auth=OAuth('', '', consumer_key, consumer_secret), format='', api_version=None)
        oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.request_token(oauth_callback = callback_url))
        write_token_file(pre_verify_token_filename, oauth_token, oauth_token_secret)

        oauth_url = 'https://api.twitter.com/oauth/authorize?' + urllib.urlencode({ 'oauth_token': oauth_token })
        return oauth_url

    # 2) We've done pre-verification, hopefully the user has authed us in Twitter
    # and we've been redirected to. Check we are and ask for the permanent tokens.
    oauth_token, oauth_token_secret = read_token_file(CREDS_PRE_VERIFIY)
    twitter = Twitter(auth=OAuth( oauth_token, oauth_token_secret, consumer_key, consumer_secret), format='', api_version=None)
    oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.access_token(oauth_verifier=oauth_verifier))
    write_token_file(verified_token_filename, oauth_token, oauth_token_secret)
    return oauth_token, oauth_token_secret

def do_tool_oauth():
    if not os.path.exists(CREDS_VERIFIED):
        if len(sys.argv) < 3:
            result = "need-oauth"
        else:
            (callback_url, oauth_verifier) = (sys.argv[1], sys.argv[2])
            result = oauth_url_dance(CONSUMER_KEY, CONSUMER_SECRET, callback_url, oauth_verifier, CREDS_PRE_VERIFIY, CREDS_VERIFIED)
        # a string means a URL for a redirect (otherwise we get a tuple back with auth tokens in)
        if type(result) == str:
            set_status_and_exit('auth-redirect', 'error', 'Permission needed from Twitter', { 'url': result } )

    oauth_token, oauth_token_secret = read_token_file(CREDS_VERIFIED)
    tw = twitter.Twitter(auth=twitter.OAuth( oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET))
    return tw

#########################################################################
# Helper functions

# Stores one Twitter user in the ScraperWiki database
def convert_user(batch, user):
    data = collections.OrderedDict()

    data['id'] = user['id']
    data['name'] = user['name']
    data['screen_name'] = user['screen_name']
    data['profile_url'] = "https://twitter.com/" + user['screen_name']
    data['profile_image'] = user['profile_image_url_https'] # shorten name to avoid wasting horizontal space

    data['description'] = user['description']
    data['location'] = user['location']
    data['url'] = user['url']

    data['followers_count'] = user['followers_count']
    data['following_count'] = user['friends_count'] # rename as "friends" is confusing to end users
    data['statuses_count'] = user['statuses_count']

    data['created_at'] = dateutil.parser.parse(user['created_at'])

    data['batch'] = batch # this is needed internally to track progress of getting all the followers

    return data

# After detecting an auth failed error mid work, call this
def clear_auth_and_restart():
    # remove auth files and respawn
    try:
        os.remove(CREDS_PRE_VERIFIY)
        os.remove(CREDS_VERIFIED)
    except OSError:
        # don't worry if the files aren't there
        pass
    subprocess.call(sys.argv)
    sys.exit()

# Signal back to the calling Javascript, to the database, and custard's status API, our status
def set_status_and_exit(status, typ, message, extra = {}):
    global current_status

    extra['status'] = status
    print json.dumps(extra)

    scraperwiki.status(typ, message)

    current_status = status
    save_status()

    sys.exit()

# Store all our progress variables
def save_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    # Update progress indicators...

    # For number of users got, we count the total of:
    # 1) all followers in the last full batch
    # 2) all followers transferred into the new batch so far
    # i.e. all those for whom batch >= (current_batch - 1)
    try:
        batch_got = scraperwiki.sql.select("count(*) as c from twitter_followers where batch >= %d" % (current_batch - 1))[0]['c']
    except:
        batch_got = 0

    data = {
        'id': 'followers',
        'current_batch': current_batch,
        'next_cursor': next_cursor,
        'batch_got': batch_got,
        'batch_expected': batch_expected,
        'current_status': current_status,
        'when': datetime.datetime.now().isoformat()

    }
    scraperwiki.sql.save(['id'], data, table_name='__status')

def get_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    try:
        data = scraperwiki.sql.select("* from __status where id='followers'")
    except sqlite3.OperationalError, e:
        if str(e) == "no such table: __status":
            return
        raise
    if len(data) == 0:
        return
    assert(len(data) == 1)
    data = data[0]

    current_batch = data['current_batch']
    next_cursor = data['next_cursor']
    batch_got = data['batch_got']
    batch_expected = data['batch_expected']
    current_status = data['current_status']

# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python
def chunks(l, n):
    return [l[i:i+n] for i in range(0, len(l), n)]

#########################################################################
# Main code

def main_function():
    # Load in all our progress variables
    current_batch = 1
    next_cursor = -1
    batch_got = 0
    batch_expected = 0
    current_status = 'clean-slate'

    pages_got = 0
    # Rename old status table to new __status name.
    # This can be removed after it has been active long enough to
    # update all existing tools.
    try :
        scraperwiki.sql.execute("SELECT 1 FROM status")
    except sqlite3.OperationalError:
       pass
    else:
        scraperwiki.sql.execute("ALTER TABLE status RENAME TO __status")

    # Parameters to this command vary:
    #   a. None: try and scrape Twitter followers
    #   b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens
    #   c. "clean-slate": wipe database and start again
    if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':
        scraperwiki.sql.execute("drop table if exists twitter_followers")
        scraperwiki.sql.execute("drop table if exists __status")
        os.system("crontab -r >/dev/null 2>&1")
        set_status_and_exit('clean-slate', 'error', 'No user set')
        sys.exit()

    # Make the followers table *first* with dumb data, calling DumpTruck directly,
    # so it appears before the status one in the list
    scraperwiki.sql.dt.create_table({'id': 1, 'batch': 1}, 'twitter_followers')

    scraperwiki.sql.execute("CREATE INDEX IF NOT EXISTS batch_index "
                            "ON twitter_followers (batch)")

    # Get user we're working on from file we store it in
    screen_name = open("user.txt").read().strip()

    # Connect to Twitter
    tw = do_tool_oauth()

    # A batch is one scan through the list of followers - we have to scan as
    # our API calls are limited.  The cursor is Twitter's identifier of where
    # in the current batch we are.
    get_status()
    # Note that each user is only in the most recent batch they've been found in
    # (we don't keep all the history)

    # Look up latest followers count
    profile = tw.users.lookup(screen_name=screen_name)
    batch_expected = profile[0]['followers_count']

    # Things basically working, so make sure we run again by writing a crontab.
    if not os.path.isfile("crontab"):
        crontab = open("tool/crontab.template").read()
        # ... run at a random minute to distribute load XXX platform should do this for us
        crontab = crontab.replace("RANDOM", str(random.randint(0, 59)))
        open("crontab", "w").write(crontab)
    os.system("crontab crontab")

    # Get as many pages in the batch as we can (most likely 15!)
    onetime = 'ONETIME' in os.environ
    live_dataset = 'LIVE_DATASET' in os.environ
    while True:
        #raise httplib.IncompleteRead('hi') # for testing
        #print "getting", next_cursor

        # get the identifiers of followers - one page worth (up to 5000 people)
        if next_cursor == -1:
            result = tw.followers.ids(screen_name=screen_name)
        else:
            result = tw.followers.ids(screen_name=screen_name, cursor=next_cursor)
        ids = result['ids']

        # and then the user details for all the ids
        double_break = False
        for chunk in chunks(ids, 100):
            users = tw.users.lookup(user_id=(",".join(map(str, chunk))))
            data = []
            for user in users:
                datum = convert_user(current_batch, user)
                data.append(datum)
            scraperwiki.sql.save(['id'], data, table_name="twitter_followers")
            save_status()

            # Don't allow more than a certain number
            if batch_got >= MAX_TO_GET:
                os.system("crontab -r >/dev/null 2>&1")
                set_status_and_exit("ok-limit", 'ok', "Reached %d follower limit" % MAX_TO_GET)

            # If being run from the user interface, return quickly after being
            # sure we've got *something* (the Javascript will then spawn us
            # again in the background to slowly get the rest)
            if onetime:
                double_break = True
                break
        if double_break:
            break

        # we have all the info for one page - record got and save it
        pages_got += 1
        next_cursor = result['next_cursor']

        # While debugging, only do one page to avoid rate limits by uncommenting this:
        # break

        if next_cursor == 0:
            # We've finished a batch
            next_cursor = -1
            current_batch += 1

            if not live_dataset:
                # Disable cron job, we're done
                os.system("crontab -r >/dev/null 2>&1")
                set_status_and_exit("ok-done", 'ok', "Finished")
            break

    # Save progress message
    set_status_and_exit("ok-updating", 'ok', "Running... %d/%d" % (batch_got, batch_expected))

try:
    main_function()
except twitter.api.TwitterHTTPError, e:
    if "Twitter sent status 401 for URL" in str(e):
        clear_auth_and_restart()

    # https://dev.twitter.com/docs/error-codes-responses
    obj = json.loads(e.response_data)
    code = obj['errors'][0]['code']
    # authentication failure
    if (code in [32, 89]):
        clear_auth_and_restart()
    # page not found
    if code == 34:
        set_status_and_exit('not-there', 'error', 'User not on Twitter')
    # rate limit exceeded
    if code == 88:
        # provided we got at least one page, rate limit isn't an error but expected
        if pages_got == 0:
            set_status_and_exit('rate-limit', 'error', 'Twitter is rate limiting you')
    else:
        # anything else is an unexpected error - if ones occur a lot, add the above instead
        raise
except httplib.IncompleteRead, e:
    # I think this is effectively a rate limit error - so only count if it was first error
    if pages_got == 0:
        set_status_and_exit('rate-limit', 'error', 'Twitter broke the connection')
<commit_msg>many logging messages<commit_after>#!/usr/bin/python

import os
import json
import urllib
import sys
import collections
import dateutil.parser
import subprocess
import httplib
import sqlite3
import datetime
import scraperwiki
import httplib
import random
import datetime
import logging

from secrets import *

MAX_TO_GET=100000
logging.basicConfig(level=logging.INFO)

# Horrendous hack to work around some Twitter / Python incompatibility
# http://bobrochel.blogspot.co.nz/2010/11/bad-servers-chunked-encoding-and.html
def patch_http_response_read(func):
    def inner(*args):
        try:
            return func(*args)
        except httplib.IncompleteRead, e:
            return e.partial

    return inner
httplib.HTTPResponse.read = patch_http_response_read(httplib.HTTPResponse.read)

# Make sure you install this version of "twitter":
# http://pypi.python.org/pypi/twitter
# http://mike.verdone.ca/twitter/
# https://github.com/sixohsix/twitter
import twitter

#########################################################################
# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which
# currently only has a function for PIN authentication, not redirect)
from twitter.api import Twitter
from twitter.oauth import OAuth, write_token_file, read_token_file
from twitter.oauth_dance import parse_oauth_tokens
def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):
    # Verification happens in two stages...

    # 1) If we haven't done a pre-verification yet... Then we get credentials from Twitter
    # that will be used to sign our redirect to them, find the redirect, and instruct the Javascript
    # that called us to do the redirect.
    if not os.path.exists(CREDS_PRE_VERIFIY):
        twitter = Twitter(auth=OAuth('', '', consumer_key, consumer_secret), format='', api_version=None)
        oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.request_token(oauth_callback = callback_url))
        write_token_file(pre_verify_token_filename, oauth_token, oauth_token_secret)

        oauth_url = 'https://api.twitter.com/oauth/authorize?' + urllib.urlencode({ 'oauth_token': oauth_token })
        return oauth_url

    # 2) We've done pre-verification, hopefully the user has authed us in Twitter
    # and we've been redirected to. Check we are and ask for the permanent tokens.
    oauth_token, oauth_token_secret = read_token_file(CREDS_PRE_VERIFIY)
    twitter = Twitter(auth=OAuth( oauth_token, oauth_token_secret, consumer_key, consumer_secret), format='', api_version=None)
    oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.access_token(oauth_verifier=oauth_verifier))
    write_token_file(verified_token_filename, oauth_token, oauth_token_secret)
    return oauth_token, oauth_token_secret

def do_tool_oauth():
    if not os.path.exists(CREDS_VERIFIED):
        if len(sys.argv) < 3:
            result = "need-oauth"
        else:
            (callback_url, oauth_verifier) = (sys.argv[1], sys.argv[2])
            result = oauth_url_dance(CONSUMER_KEY, CONSUMER_SECRET, callback_url, oauth_verifier, CREDS_PRE_VERIFIY, CREDS_VERIFIED)
        # a string means a URL for a redirect (otherwise we get a tuple back with auth tokens in)
        if type(result) == str:
            set_status_and_exit('auth-redirect', 'error', 'Permission needed from Twitter', { 'url': result } )

    oauth_token, oauth_token_secret = read_token_file(CREDS_VERIFIED)
    tw = twitter.Twitter(auth=twitter.OAuth( oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET))
    return tw

#########################################################################
# Helper functions

# Stores one Twitter user in the ScraperWiki database
def convert_user(batch, user):
    data = collections.OrderedDict()

    data['id'] = user['id']
    data['name'] = user['name']
    data['screen_name'] = user['screen_name']
    data['profile_url'] = "https://twitter.com/" + user['screen_name']
    data['profile_image'] = user['profile_image_url_https'] # shorten name to avoid wasting horizontal space

    data['description'] = user['description']
    data['location'] = user['location']
    data['url'] = user['url']

    data['followers_count'] = user['followers_count']
    data['following_count'] = user['friends_count'] # rename as "friends" is confusing to end users
    data['statuses_count'] = user['statuses_count']

    data['created_at'] = dateutil.parser.parse(user['created_at'])

    data['batch'] = batch # this is needed internally to track progress of getting all the followers

    return data

# After detecting an auth failed error mid work, call this
def clear_auth_and_restart():
    # remove auth files and respawn
    try:
        os.remove(CREDS_PRE_VERIFIY)
        os.remove(CREDS_VERIFIED)
    except OSError:
        # don't worry if the files aren't there
        pass
    subprocess.call(sys.argv)
    sys.exit()

# Signal back to the calling Javascript, to the database, and custard's status API, our status
def set_status_and_exit(status, typ, message, extra = {}):
    global current_status

    extra['status'] = status
    print json.dumps(extra)

    scraperwiki.status(typ, message)

    current_status = status
    save_status()

    sys.exit()

# Store all our progress variables
def save_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    # Update progress indicators...

    # For number of users got, we count the total of:
    # 1) all followers in the last full batch
    # 2) all followers transferred into the new batch so far
    # i.e. all those for whom batch >= (current_batch - 1)
    try:
        batch_got = scraperwiki.sql.select("count(*) as c from twitter_followers where batch >= %d" % (current_batch - 1))[0]['c']
    except:
        batch_got = 0

    data = {
        'id': 'followers',
        'current_batch': current_batch,
        'next_cursor': next_cursor,
        'batch_got': batch_got,
        'batch_expected': batch_expected,
        'current_status': current_status,
        'when': datetime.datetime.now().isoformat()

    }
    scraperwiki.sql.save(['id'], data, table_name='__status')

def get_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    try:
        data = scraperwiki.sql.select("* from __status where id='followers'")
    except sqlite3.OperationalError, e:
        if str(e) == "no such table: __status":
            return
        raise
    if len(data) == 0:
        return
    assert(len(data) == 1)
    data = data[0]

    current_batch = data['current_batch']
    next_cursor = data['next_cursor']
    batch_got = data['batch_got']
    batch_expected = data['batch_expected']
    current_status = data['current_status']
    return data

# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python
def chunks(l, n):
    return [l[i:i+n] for i in range(0, len(l), n)]

def move_legacy_table():
    # Rename old status table to new __status name.
    # This can be removed after it has been active long enough to
    # update all existing tools.
    try :
        scraperwiki.sql.execute("SELECT 1 FROM status")
    except sqlite3.OperationalError:
        logging.info("No legacy status table detected.")
        pass
    else:
        logging.warn("Legacy status table detected.")
        scraperwiki.sql.execute("ALTER TABLE status RENAME TO __status")

#########################################################################
# Main code

def main_function():
    logging.info("main_function()")
    # Load in all our progress variables
    current_batch = 1
    next_cursor = -1
    batch_got = 0
    batch_expected = 0
    current_status = 'clean-slate'

    pages_got = 0
    # Rename old status table to new __status name.
    # This can be removed after it has been active long enough to
    # update all existing tools.
    move_legacy_table()

    # Parameters to this command vary:
    #   a. None: try and scrape Twitter followers
    #   b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens
    #   c. "clean-slate": wipe database and start again
    if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':
        logging.warn("Cleaning slate")
        scraperwiki.sql.execute("drop table if exists twitter_followers")
        scraperwiki.sql.execute("drop table if exists __status")
        os.system("crontab -r >/dev/null 2>&1")
        set_status_and_exit('clean-slate', 'error', 'No user set')
        sys.exit()

    # Make the followers table *first* with dumb data, calling DumpTruck directly,
    # so it appears before the status one in the list
    scraperwiki.sql.dt.create_table({'id': 1, 'batch': 1}, 'twitter_followers')

    scraperwiki.sql.execute("CREATE INDEX IF NOT EXISTS batch_index "
                            "ON twitter_followers (batch)")

    # Get user we're working on from file we store it in
    screen_name = open("user.txt").read().strip()

    # Connect to Twitter
    tw = do_tool_oauth()
    logging.info("Authenticated")
    # A batch is one scan through the list of followers - we have to scan as
    # our API calls are limited.  The cursor is Twitter's identifier of where
    # in the current batch we are.
    status_info = get_status()
    logging.info("Loaded status: {!r}".format(status_info))

    # Note that each user is only in the most recent batch they've been found in
    # (we don't keep all the history)

    # Look up latest followers count
    profile = tw.users.lookup(screen_name=screen_name)
    logging.debug("User details: {!r}".format(profile))
    batch_expected = profile[0]['followers_count']
    logging.info("Batch expected: {!r}".format(batch_expected))

    # Things basically working, so make sure we run again by writing a crontab.
    if not os.path.isfile("crontab"):
        logging.warn("Crontab not detected. Installing...")
        crontab = open("tool/crontab.template").read()
        # ... run at a random minute to distribute load XXX platform should do this for us
        crontab = crontab.replace("RANDOM", str(random.randint(0, 59)))
        open("crontab", "w").write(crontab)
    else:
        logging.info("Crontab present.")
    os.system("crontab crontab")

    # Get as many pages in the batch as we can (most likely 15!)
    onetime = 'ONETIME' in os.environ
    live_dataset = 'LIVE_DATASET' in os.environ
    logging.info("Is onetime: {!r}".format(onetime))
    logging.info("Is Live: {!r}".format(live_dataset))

    while True:
        #raise httplib.IncompleteRead('hi') # for testing
        #print "getting", next_cursor

        # get the identifiers of followers - one page worth (up to 5000 people)
        logging.info("next_cursor: {!r}".format(next_cursor))
        if next_cursor == -1:
            result = tw.followers.ids(screen_name=screen_name)
        else:
            result = tw.followers.ids(screen_name=screen_name, cursor=next_cursor)
        ids = result['ids']
        logging.info("processing ids {!r}".format(ids))

        # and then the user details for all the ids
        double_break = False
        for chunk in chunks(ids, 100):
            users = tw.users.lookup(user_id=(",".join(map(str, chunk))))
            data = []
            for user in users:
                datum = convert_user(current_batch, user)
                data.append(datum)
            scraperwiki.sql.save(['id'], data, table_name="twitter_followers")
            save_status()

            # Don't allow more than a certain number
            if batch_got >= MAX_TO_GET:
                os.system("crontab -r >/dev/null 2>&1")
                set_status_and_exit("ok-limit", 'ok', "Reached %d follower limit" % MAX_TO_GET)

            # If being run from the user interface, return quickly after being
            # sure we've got *something* (the Javascript will then spawn us
            # again in the background to slowly get the rest)
            if onetime:
                double_break = True
                break
        if double_break:
            break
        logging.info("... ok")

        # we have all the info for one page - record got and save it
        pages_got += 1
        next_cursor = result['next_cursor']

        # While debugging, only do one page to avoid rate limits by uncommenting this:
        # break

        if next_cursor == 0:
            logging.warn("Excellent! We've finished a batch!")
            # We've finished a batch
            next_cursor = -1
            current_batch += 1

            if not live_dataset:
                # Disable cron job, we're done
                logging.warn("All done: disabling cronjob")
                os.system("crontab -r >/dev/null 2>&1")
                set_status_and_exit("ok-done", 'ok', "Finished")
            break

    # Save progress message
    logging.info("We'll come back later. Bye for now.")
    set_status_and_exit("ok-updating", 'ok', "Running... %d/%d" % (batch_got, batch_expected))

try:
    main_function()
except twitter.api.TwitterHTTPError, e:
    if "Twitter sent status 401 for URL" in str(e):
        clear_auth_and_restart()

    # https://dev.twitter.com/docs/error-codes-responses
    obj = json.loads(e.response_data)
    code = obj['errors'][0]['code']
    # authentication failure
    if (code in [32, 89]):
        clear_auth_and_restart()
    # page not found
    if code == 34:
        set_status_and_exit('not-there', 'error', 'User not on Twitter')
    # rate limit exceeded
    if code == 88:
        # provided we got at least one page, rate limit isn't an error but expected
        if pages_got == 0:
            set_status_and_exit('rate-limit', 'error', 'Twitter is rate limiting you')
    else:
        # anything else is an unexpected error - if ones occur a lot, add the above instead
        raise
except httplib.IncompleteRead, e:
    # I think this is effectively a rate limit error - so only count if it was first error
    if pages_got == 0:
        set_status_and_exit('rate-limit', 'error', 'Twitter broke the connection')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Export Layers.
#
# Copyright (C) 2013-2018 khalim19 <khalim19@gmail.com>
#
# Export Layers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Export Layers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Export Layers.  If not, see <https://www.gnu.org/licenses/>.

from __future__ import absolute_import, division, print_function, unicode_literals
from future.builtins import *

import inspect
import os
import shutil
import unittest

from gimp import pdb

from export_layers import pygimplib
from export_layers.pygimplib import pgfileformats
from export_layers.pygimplib import pgitemtree
from export_layers.pygimplib import pgpath
from export_layers.pygimplib import pgpdb
from export_layers.pygimplib import pgutils

from .. import config
config.init()

from .. import exportlayers
from .. import settings_plugin

pygimplib.init()

#===============================================================================

_CURRENT_MODULE_DIRPATH = os.path.dirname(pgutils.get_current_module_filepath())
TEST_IMAGES_DIRPATH = os.path.join(_CURRENT_MODULE_DIRPATH, "test_images")

EXPECTED_RESULTS_DIRPATH = os.path.join(TEST_IMAGES_DIRPATH, "expected_results")
OUTPUT_DIRPATH = os.path.join(TEST_IMAGES_DIRPATH, "temp_output")
INCORRECT_RESULTS_DIRPATH = os.path.join(TEST_IMAGES_DIRPATH, "incorrect_results")

#===============================================================================

class TestExportLayersCompareLayerContents(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    pdb.gimp_context_push()

    cls.test_image_filepath = os.path.join(
      TEST_IMAGES_DIRPATH, "test_export_layers_contents.xcf")
    cls.test_image = cls._load_image()

    cls.output_dirpath = OUTPUT_DIRPATH

    if os.path.exists(cls.output_dirpath):
      shutil.rmtree(cls.output_dirpath)

    if os.path.exists(INCORRECT_RESULTS_DIRPATH):
      shutil.rmtree(INCORRECT_RESULTS_DIRPATH)

    cls.default_expected_layers_dirpath = EXPECTED_RESULTS_DIRPATH
    # key: path to directory containing expected results
    # value: gimp.Image instance
    cls.expected_images = {}

  @classmethod
  def tearDownClass(cls):
    pdb.gimp_image_delete(cls.test_image)
    for image in cls.expected_images.values():
      pdb.gimp_image_delete(image)

    pdb.gimp_context_pop()
    pdb.gimp_progress_end()

  def setUp(self):
    self.image_with_results = None

  def tearDown(self):
    if self.image_with_results is not None:
      pdb.gimp_image_delete(self.image_with_results)

    if os.path.exists(self.output_dirpath):
      shutil.rmtree(self.output_dirpath)

  def test_default_settings(self):
    self.compare()

  def test_ignore_layer_modes(self):
    self.compare(
      {"operations/ignore_layer_modes": True})

  def test_autocrop(self):
    self.compare(
      {"operations/autocrop": True},
      [("left-frame-with-extra-borders", "left-frame-with-extra-borders_autocrop"),
       ("main-background", "main-background_autocrop")])

  def test_use_image_size(self):
    self.compare(
      {"use_image_size": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "use_image_size"))

  def test_use_image_size_autocrop(self):
    self.compare(
      {"use_image_size": True,
       "operations/autocrop": True},
      [("left-frame-with-extra-borders", "left-frame-with-extra-borders_autocrop"),
       ("main-background", "main-background_autocrop")],
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "use_image_size"))

  def test_background(self):
    self.compare(
      {"operations/insert_background_layers": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background"))

  def test_background_autocrop(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop": True},
      [("main-background", "main-background_autocrop"),
       ("overlay", "overlay_background"),
       ("bottom-frame-semi-transparent",
        "bottom-frame-semi-transparent_background_autocrop"),
       ("left-frame-with-extra-borders",
        "left-frame-with-extra-borders_autocrop")])

  def test_background_autocrop_use_image_size(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop": True,
       "use_image_size": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background", "autocrop-use_image_size"))

  def test_background_autocrop_background(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop_background": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background"))

  def test_background_autocrop_background_use_image_size(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop_background": True,
       "use_image_size": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background",
        "autocrop_background-use_image_size"))

  def test_foreground(self):
    layer_tree = pgitemtree.LayerTree(
      self.test_image, name=pygimplib.config.SOURCE_PERSISTENT_NAME)
    for layer_elem in layer_tree:
      if "background" in layer_elem.tags:
        layer_elem.remove_tag("background")
        layer_elem.add_tag("foreground")

    self.compare(
      {"operations/insert_foreground_layers": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "foreground"))

    self._reload_image()

  def compare(
        self, different_settings=None, different_results_and_expected_layers=None,
        expected_results_dirpath=None):
    settings = settings_plugin.create_settings()
    settings["special"]["image"].set_value(self.test_image)
    settings["main"]["output_directory"].set_value(self.output_dirpath)

    if different_settings is not None:
      for setting_name, setting_value in different_settings.items():
        settings["main"][setting_name].set_value(setting_value)

    if expected_results_dirpath is None:
      expected_results_dirpath = self.default_expected_layers_dirpath

    if expected_results_dirpath not in self.expected_images:
      self.expected_images[expected_results_dirpath], expected_layers = (
        self._load_layers_from_dirpath(expected_results_dirpath))
    else:
      expected_layers = {
        layer.name: layer
        for layer in self.expected_images[expected_results_dirpath].layers}

    self._export(settings)

    self.image_with_results, layers = self._load_layers_from_dirpath(self.output_dirpath)

    if different_results_and_expected_layers is not None:
      for layer_name, expected_layer_name in different_results_and_expected_layers:
        expected_layers[layer_name] = expected_layers[expected_layer_name]

    for layer in layers.values():
      test_case_name = inspect.stack()[1][-3]
      self._compare_layers(
        layer, expected_layers[layer.name], settings, test_case_name,
        expected_results_dirpath)

  @staticmethod
  def _export(settings):
    for operation_setting in settings.walk():
      if exportlayers.is_valid_operation(operation_setting):
        exportlayers.add_operation(operation_setting)

    layer_exporter = exportlayers.LayerExporter(
      settings["special"]["run_mode"].value, settings["special"]["image"].value,
      settings["main"])

    layer_exporter.export()

    for operation_setting in settings.walk():
      if exportlayers.is_valid_operation(operation_setting):
        exportlayers.remove_operation(operation_setting)

  def _compare_layers(
        self, layer, expected_layer, settings, test_case_name, expected_results_dirpath):
    if not pgpdb.compare_layers([layer, expected_layer]):
      self._save_incorrect_layers(
        layer, expected_layer, settings, test_case_name, expected_results_dirpath)

    self.assertEqual(
      pgpdb.compare_layers([layer, expected_layer]), True,
      msg=("Layers are not identical:\nprocessed layer: {}\nexpected layer: {}".format(
        layer.name, expected_layer.name)))

  def _save_incorrect_layers(
        self, layer, expected_layer, settings, test_case_name, expected_results_dirpath):
    incorrect_layers_dirpath = os.path.join(INCORRECT_RESULTS_DIRPATH, test_case_name)
    pgpath.make_dirs(incorrect_layers_dirpath)

    self._copy_incorrect_layer(
      layer, settings, self.output_dirpath, incorrect_layers_dirpath, "_actual")
    self._copy_incorrect_layer(
      expected_layer, settings, expected_results_dirpath,
      incorrect_layers_dirpath, "_expected")

  @staticmethod
  def _copy_incorrect_layer(
        layer, settings, layer_dirpath, incorrect_layers_dirpath, filename_suffix):
    layer_input_filename = "{}.{}".format(
      layer.name, settings["main/file_extension"].value)
    layer_output_filename = "{}{}.{}".format(
      layer.name, filename_suffix, settings["main/file_extension"].value)

    shutil.copy(
      os.path.join(layer_dirpath, layer_input_filename),
      os.path.join(incorrect_layers_dirpath, layer_output_filename))

  @classmethod
  def _load_image(cls):
    return pdb.gimp_file_load(
      cls.test_image_filepath, os.path.basename(cls.test_image_filepath))

  @classmethod
  def _reload_image(cls):
    pdb.gimp_image_delete(cls.test_image)
    cls.test_image = cls._load_image()

  @classmethod
  def _load_layers_from_dirpath(cls, layers_dirpath):
    return cls._load_layers(cls._list_layer_filepaths(layers_dirpath))

  @staticmethod
  def _load_layers(layer_filepaths):
    """
    Load layers from specified file paths into a new image. Return the image and
    a dict with (layer name: gimp.Layer instance) pairs.
    """

    image = pgpdb.load_layers(layer_filepaths, image=None, strip_file_extension=True)
    return image, {layer.name: layer for layer in image.layers}

  @staticmethod
  def _list_layer_filepaths(layers_dirpath):
    layers_filepaths = []

    for filename in os.listdir(layers_dirpath):
      path = os.path.join(layers_dirpath, filename)
      if os.path.isfile(path):
        layers_filepaths.append(path)

    return layers_filepaths

#===============================================================================

def test_file_formats(layer_exporter, export_settings):
  orig_output_dirpath = export_settings["output_directory"].value

  for file_format in pgfileformats.file_formats:
    for file_extension in file_format.file_extensions:
      export_settings["file_extension"].set_value(file_extension)
      export_settings["output_directory"].set_value(
        os.path.join(orig_output_dirpath, file_extension))
      try:
        layer_exporter.export()
      except exportlayers.ExportLayersError:
        # Do not stop execution when one file format causes an error.
        continue
<commit_msg>test_exportlayers: Rename variables for consistency<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Export Layers.
#
# Copyright (C) 2013-2018 khalim19 <khalim19@gmail.com>
#
# Export Layers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Export Layers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Export Layers.  If not, see <https://www.gnu.org/licenses/>.

from __future__ import absolute_import, division, print_function, unicode_literals
from future.builtins import *

import inspect
import os
import shutil
import unittest

from gimp import pdb

from export_layers import pygimplib
from export_layers.pygimplib import pgfileformats
from export_layers.pygimplib import pgitemtree
from export_layers.pygimplib import pgpath
from export_layers.pygimplib import pgpdb
from export_layers.pygimplib import pgutils

from .. import config
config.init()

from .. import exportlayers
from .. import settings_plugin

pygimplib.init()

#===============================================================================

_CURRENT_MODULE_DIRPATH = os.path.dirname(pgutils.get_current_module_filepath())
TEST_IMAGES_DIRPATH = os.path.join(_CURRENT_MODULE_DIRPATH, "test_images")

EXPECTED_RESULTS_DIRPATH = os.path.join(TEST_IMAGES_DIRPATH, "expected_results")
OUTPUT_DIRPATH = os.path.join(TEST_IMAGES_DIRPATH, "temp_output")
INCORRECT_RESULTS_DIRPATH = os.path.join(TEST_IMAGES_DIRPATH, "incorrect_results")

#===============================================================================

class TestExportLayersCompareLayerContents(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    pdb.gimp_context_push()

    cls.test_image_filepath = os.path.join(
      TEST_IMAGES_DIRPATH, "test_export_layers_contents.xcf")
    cls.test_image = cls._load_image()

    cls.output_dirpath = OUTPUT_DIRPATH

    if os.path.exists(cls.output_dirpath):
      shutil.rmtree(cls.output_dirpath)

    if os.path.exists(INCORRECT_RESULTS_DIRPATH):
      shutil.rmtree(INCORRECT_RESULTS_DIRPATH)

    cls.default_expected_layers_dirpath = EXPECTED_RESULTS_DIRPATH
    # key: path to directory containing expected results
    # value: gimp.Image instance
    cls.expected_images = {}

  @classmethod
  def tearDownClass(cls):
    pdb.gimp_image_delete(cls.test_image)
    for image in cls.expected_images.values():
      pdb.gimp_image_delete(image)

    pdb.gimp_context_pop()
    pdb.gimp_progress_end()

  def setUp(self):
    self.image_with_results = None

  def tearDown(self):
    if self.image_with_results is not None:
      pdb.gimp_image_delete(self.image_with_results)

    if os.path.exists(self.output_dirpath):
      shutil.rmtree(self.output_dirpath)

  def test_default_settings(self):
    self.compare()

  def test_ignore_layer_modes(self):
    self.compare(
      {"operations/ignore_layer_modes": True})

  def test_autocrop(self):
    self.compare(
      {"operations/autocrop": True},
      [("left-frame-with-extra-borders", "left-frame-with-extra-borders_autocrop"),
       ("main-background", "main-background_autocrop")])

  def test_use_image_size(self):
    self.compare(
      {"use_image_size": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "use_image_size"))

  def test_use_image_size_autocrop(self):
    self.compare(
      {"use_image_size": True,
       "operations/autocrop": True},
      [("left-frame-with-extra-borders", "left-frame-with-extra-borders_autocrop"),
       ("main-background", "main-background_autocrop")],
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "use_image_size"))

  def test_background(self):
    self.compare(
      {"operations/insert_background_layers": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background"))

  def test_background_autocrop(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop": True},
      [("main-background", "main-background_autocrop"),
       ("overlay", "overlay_background"),
       ("bottom-frame-semi-transparent",
        "bottom-frame-semi-transparent_background_autocrop"),
       ("left-frame-with-extra-borders",
        "left-frame-with-extra-borders_autocrop")])

  def test_background_autocrop_use_image_size(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop": True,
       "use_image_size": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background", "autocrop-use_image_size"))

  def test_background_autocrop_background(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop_background": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background"))

  def test_background_autocrop_background_use_image_size(self):
    self.compare(
      {"operations/insert_background_layers": True,
       "operations/autocrop_background": True,
       "use_image_size": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "background",
        "autocrop_background-use_image_size"))

  def test_foreground(self):
    layer_tree = pgitemtree.LayerTree(
      self.test_image, name=pygimplib.config.SOURCE_PERSISTENT_NAME)
    for layer_elem in layer_tree:
      if "background" in layer_elem.tags:
        layer_elem.remove_tag("background")
        layer_elem.add_tag("foreground")

    self.compare(
      {"operations/insert_foreground_layers": True},
      expected_results_dirpath=os.path.join(
        self.default_expected_layers_dirpath, "foreground"))

    self._reload_image()

  def compare(
        self, different_settings=None, different_results_and_expected_layers=None,
        expected_results_dirpath=None):
    settings = settings_plugin.create_settings()
    settings["special"]["image"].set_value(self.test_image)
    settings["main"]["output_directory"].set_value(self.output_dirpath)

    if different_settings is not None:
      for setting_name, setting_value in different_settings.items():
        settings["main"][setting_name].set_value(setting_value)

    if expected_results_dirpath is None:
      expected_results_dirpath = self.default_expected_layers_dirpath

    if expected_results_dirpath not in self.expected_images:
      self.expected_images[expected_results_dirpath], expected_layers = (
        self._load_layers_from_dirpath(expected_results_dirpath))
    else:
      expected_layers = {
        layer.name: layer
        for layer in self.expected_images[expected_results_dirpath].layers}

    self._export(settings)

    self.image_with_results, layers = self._load_layers_from_dirpath(self.output_dirpath)

    if different_results_and_expected_layers is not None:
      for layer_name, expected_layer_name in different_results_and_expected_layers:
        expected_layers[layer_name] = expected_layers[expected_layer_name]

    for layer in layers.values():
      test_case_name = inspect.stack()[1][-3]
      self._compare_layers(
        layer, expected_layers[layer.name], settings, test_case_name,
        expected_results_dirpath)

  @staticmethod
  def _export(settings):
    for operation_setting in settings.walk():
      if exportlayers.is_valid_operation(operation_setting):
        exportlayers.add_operation(operation_setting)

    layer_exporter = exportlayers.LayerExporter(
      settings["special"]["run_mode"].value, settings["special"]["image"].value,
      settings["main"])

    layer_exporter.export()

    for operation_setting in settings.walk():
      if exportlayers.is_valid_operation(operation_setting):
        exportlayers.remove_operation(operation_setting)

  def _compare_layers(
        self, layer, expected_layer, settings, test_case_name, expected_results_dirpath):
    if not pgpdb.compare_layers([layer, expected_layer]):
      self._save_incorrect_layers(
        layer, expected_layer, settings, test_case_name, expected_results_dirpath)

    self.assertEqual(
      pgpdb.compare_layers([layer, expected_layer]), True,
      msg=("Layers are not identical:\nprocessed layer: {}\nexpected layer: {}".format(
        layer.name, expected_layer.name)))

  def _save_incorrect_layers(
        self, layer, expected_layer, settings, test_case_name, expected_results_dirpath):
    incorrect_layers_dirpath = os.path.join(INCORRECT_RESULTS_DIRPATH, test_case_name)
    pgpath.make_dirs(incorrect_layers_dirpath)

    self._copy_incorrect_layer(
      layer, settings, self.output_dirpath, incorrect_layers_dirpath, "_actual")
    self._copy_incorrect_layer(
      expected_layer, settings, expected_results_dirpath,
      incorrect_layers_dirpath, "_expected")

  @staticmethod
  def _copy_incorrect_layer(
        layer, settings, layer_dirpath, incorrect_layers_dirpath, filename_suffix):
    layer_input_filename = "{}.{}".format(
      layer.name, settings["main/file_extension"].value)
    layer_output_filename = "{}{}.{}".format(
      layer.name, filename_suffix, settings["main/file_extension"].value)

    shutil.copy(
      os.path.join(layer_dirpath, layer_input_filename),
      os.path.join(incorrect_layers_dirpath, layer_output_filename))

  @classmethod
  def _load_image(cls):
    return pdb.gimp_file_load(
      cls.test_image_filepath, os.path.basename(cls.test_image_filepath))

  @classmethod
  def _reload_image(cls):
    pdb.gimp_image_delete(cls.test_image)
    cls.test_image = cls._load_image()

  @classmethod
  def _load_layers_from_dirpath(cls, layers_dirpath):
    return cls._load_layers(cls._list_layer_filepaths(layers_dirpath))

  @staticmethod
  def _load_layers(layer_filepaths):
    """
    Load layers from specified file paths into a new image. Return the image and
    a dict with (layer name: gimp.Layer instance) pairs.
    """

    image = pgpdb.load_layers(layer_filepaths, image=None, strip_file_extension=True)
    return image, {layer.name: layer for layer in image.layers}

  @staticmethod
  def _list_layer_filepaths(layers_dirpath):
    layer_filepaths = []

    for filename in os.listdir(layers_dirpath):
      path = os.path.join(layers_dirpath, filename)
      if os.path.isfile(path):
        layer_filepaths.append(path)

    return layer_filepaths

#===============================================================================

def test_file_formats(layer_exporter, export_settings):
  orig_output_dirpath = export_settings["output_directory"].value

  for file_format in pgfileformats.file_formats:
    for file_extension in file_format.file_extensions:
      export_settings["file_extension"].set_value(file_extension)
      export_settings["output_directory"].set_value(
        os.path.join(orig_output_dirpath, file_extension))
      try:
        layer_exporter.export()
      except exportlayers.ExportLayersError:
        # Do not stop execution when one file format causes an error.
        continue
<|endoftext|>


<commit_before>import numpy as np
from numpy import min, max, float, array, float64
#from PyscesToolBox import PyscesToolBox as PYCtools
from sympy import Symbol
#from IPython.display import Latex
from ...utils.misc import silence_print
from ...utils.plotting import Data2D
from ... import modeltools
from pysces import ModelMap

def cctype(obj):
    return 'ccobjects' in str(type(obj))

@silence_print
def get_state(mod, do_state=False):
    if do_state:
        mod.doState()
    ss = [getattr(mod, 'J_' + r) for r in mod.reactions] + \
        [getattr(mod, s + '_ss') for s in mod.species]
    return ss

@silence_print
def silent_mca(mod):
    mod.doMca()

def get_value_eval(expression,subs_dict):
    for k,v in subs_dict.iteritems():
        subs_dict[k] = float64(v)
    ans = eval(expression,{},subs_dict)
    return ans

class StateKeeper:
    def __init__(self, state):
        self._last_state_for_mca = state

    def do_mca_state(self, mod, state):
        if state != self._last_state_for_mca:
            silent_mca(mod)
            self._last_state_for_mca = state

class CCBase(object):

    """The base object for the control coefficients and control patterns"""

    def __init__(self, mod, name, expression, ltxe, state_keeper):
        super(CCBase, self).__init__()

        self.expression = expression
        self.mod = mod
        self._ltxe = ltxe
        self._state_keeper = state_keeper
        self.name = name
        self._latex_name = '\\Sigma'
        self._str_expression = str(self.expression)

        self._value = None
        self._latex_expression = None
        self._state_ = get_state(mod)

    @property
    def latex_expression(self):
        if not self._latex_expression:
            self._latex_expression = self._ltxe.expression_to_latex(
                self.expression
            )
        return self._latex_expression

    @property
    def latex_name(self):
        return self._latex_name

    @property
    def value(self):
        """The value property. Calls self._calc_value() when self._value
        is None and returns self._value"""
        new_ss = get_state(self.mod)
        state_changed = new_ss != self._state_
        if state_changed:
            self._state_keeper.do_mca_state(self.mod, new_ss)
            self._calc_value()
            self._state_ = new_ss
        elif not self._value:
            self._calc_value()
        return self._value

    def _repr_latex_(self):
        return '$%s = %s = %.3f$' % (self.latex_name,
                                     self.latex_expression,
                                     self.value)

    def _calc_value(self):
        """Calculates the value of the expression"""
        keys = self.expression.atoms(Symbol)
        subsdict = {}
        for key in keys:
            str_key = str(key)
            subsdict[str_key] = getattr(self.mod, str_key)
        self._value = get_value_eval(self._str_expression,subsdict)

    def __repr__(self):
        return self.expression.__repr__()

    def __add__(self, other):
        if cctype(other):
            return self.expression.__add__(other.expression)
        else:
            return self.expression.__add__(other)

    def __mul__(self, other):
        if cctype(other):
            return self.expression.__mul__(other.expression)
        else:
            return self.expression.__mul__(other)

    def __div__(self, other):
        if cctype(other):
            return self.expression.__div__(other.expression)
        else:
            return self.expression.__div__(other)

    def __pow__(self, other):
        if cctype(other):
            return self.expression.__pow__(other.expression)
        else:
            return self.expression.__pow__(other)

class CCoef(CCBase):

    """The object the stores control coefficients. Inherits from CCBase"""

    def __init__(self, mod, name, expression, denominator, ltxe, state_keeper):
        super(CCoef, self).__init__(mod, name, expression, ltxe, state_keeper)
        self.numerator = expression
        self.denominator = denominator.expression
        self.expression = self.numerator / denominator.expression
        self.denominator_object = denominator

        self._latex_numerator = None
        self._latex_expression_full = None
        self._latex_expression = None
        self._latex_name = None

        self.control_patterns = None

        self._set_control_patterns()

    @property
    def latex_numerator(self):
        if not self._latex_numerator:
            self._latex_numerator = self._ltxe.expression_to_latex(
                self.numerator
            )
        return self._latex_numerator

    @property
    def latex_expression_full(self):
        if not self._latex_expression_full:
            full_expr = '\\frac{' + self.latex_numerator + '}{' \
                + self.denominator_object.latex_expression + '}'
            self._latex_expression_full = full_expr
        return self._latex_expression_full

    @property
    def latex_expression(self):
        if not self._latex_expression:
            self._latex_expression = '(' + self.latex_numerator + ')' + '/ \\,\\Sigma'
        return self._latex_expression

    @property
    def latex_name(self):
        if not self._latex_name:
            self._latex_name = self._ltxe.expression_to_latex(
                self.name
            )
        return self._latex_name

    #@property
    # def control_patterns(self):
    #    if not self._control_patterns:
    #        self._set_control_patterns()
    #    return self._control_patterns

    def parscan(self, parameter, scan_range, scan_type='percentage', init_return=False):
        """Performs a parameter scan and returns numpy array object
           with the parameter values in the first column and
           percentage contribution of each control pattern
           in subsequent columns

           Arguments:
           parameter   --  the parameter of the model to scan
           scan_range  --  the range across which to scan 'parameter'

           calls self._recalculate_value() for each value of
           parameter in scan_range"""

        assert scan_type in ['percentage', 'value']
        if scan_type is 'percentage':
            scan_res = [list() for i in range(len(self.control_patterns) + 1)]
        elif scan_type is 'value':
            scan_res = [list() for i in range(len(self.control_patterns) + 2)]
        scan_res[0] = scan_range

        # print type(scan_res)
        init = getattr(self.mod, parameter)
        for parvalue in scan_range:
            setattr(self.mod, parameter, parvalue)
            self.mod.SetQuiet()
            self.mod.doMca()
            self.mod.SetLoud()

            for i, cp in enumerate(self.control_patterns):
                # print type(scan_res[i+1])
                if scan_type is 'percentage':
                    scan_res[i + 1].append(cp.percentage)
                elif scan_type is 'value':
                    scan_res[i + 1].append(cp.value)

            if scan_type is 'value':
                scan_res[i + 2].append(self.value)

        if init_return:
            self.mod.SetQuiet()
            setattr(self.mod, parameter, init)
            self.mod.doMca()
            self.mod.SetLoud()

        if scan_type is 'value':
            column_names = [parameter] + [cp.name for cp in self.control_patterns] + [self.name]
            y_label = 'Control coefficient/pattern value'
        if scan_type is 'percentage':
            column_names = [parameter] + [cp.name for cp in self.control_patterns]
            y_label = 'Percentage Contribution'

        mm = ModelMap(self.mod)
        species = mm.hasSpecies()
        if parameter in species:
            x_label = '[%s]' % parameter.replace('_', ' ')
        else:
            x_label = parameter
        ax_properties = {'ylabel':y_label,
                         'xlabel':x_label,
                         'xscale': 'linear',
                         'yscale': 'linear',
                         'xlim': [scan_range[0],scan_range[-1]]}
        data_array = array(scan_res, dtype=np.float).transpose()
        data = Data2D(self.mod, column_names, data_array, self._ltxe, 'symca', ax_properties)
        return data

    def _recalculate_value(self):
        """Recalculates the control coefficients and control pattern
           values. calls _calc_value() for self and each control
           pattern. Useful for when model parameters change"""
        keys = self.expression.atoms(Symbol)
        subsdict = {}
        for key in keys:
            str_key = str(key)
            subsdict[str_key] = getattr(self.mod, str_key)
        for pattern in self.control_patterns:
            pattern._calc_value(subsdict)
        self._calc_value()

    def _calc_value(self):
        """Calculates the numeric value of the control pattern from the
           values of its control patterns."""
        self._value = sum([pattern.value for pattern in self.control_patterns])

    def _set_control_patterns(self):
        """Divides control coefficient into control patterns and saves
           results in self.CPx where x is a number is the number of the
           control pattern as it appears in in control coefficient
           expression"""
        patterns = self.numerator.as_coeff_add()[1]
        cps = []
        for i, pattern in enumerate(patterns):
            name = 'CP' + str(1 + i)
            cp = CPattern(self.mod,
                          name,
                          pattern,
                          self.denominator_object,
                          self,
                          self._ltxe,
                          self._state_keeper)
            setattr(self, name, cp)
            cps.append(cp)
        self.control_patterns = cps
        #assert self._check_control_patterns == True

    def _check_control_patterns(self):
        """Checks that all control patterns are either positive or negative"""
        all_same = False
        poscomp = [i.value > 0 for i in self.control_patterns]
        negcomp = [i.value < 0 for i in self.control_patterns]
        if all(poscomp):
            all_same = True
        elif all(negcomp):
            all_same = True
        return all_same

class CPattern(CCBase):

    """docstring for CPattern"""

    def __init__(self,
                 mod,
                 name,
                 expression,
                 denominator,
                 parent,
                 ltxe,
                 state_keeper):
        super(CPattern, self).__init__(mod,
                                       name,
                                       expression,
                                       ltxe,
                                       state_keeper)
        self.numerator = expression
        self.denominator = denominator.expression
        self.expression = self.numerator / denominator.expression
        self.denominator_object = denominator
        self.parent = parent
        self._str_expression = str(self.expression)

        self._latex_numerator = None
        self._latex_expression_full = None
        self._latex_expression = None
        self._latex_name = None
        self._percentage = None

    def _calc_value(self,subsdict=None):
        """Calculates the value of the expression"""
        if subsdict is None:
            keys = self.expression.atoms(Symbol)
            subsdict = {}
            for key in keys:
                str_key = str(key)
                subsdict[str_key] = getattr(self.mod, str_key)
        self._value = get_value_eval(self._str_expression,subsdict)

    @property
    def latex_numerator(self):
        if not self._latex_numerator:
            self._latex_numerator = self._ltxe.expression_to_latex(
                self.numerator
            )
        return self._latex_numerator

    @property
    def latex_expression_full(self):
        if not self._latex_expression_full:
            full_expr = '\\frac{' + self.latex_numerator + '}{' \
                + self.denominator_object.latex_expression + '}'
            self._latex_expression_full = full_expr
        return self._latex_expression_full

    @property
    def latex_expression(self):
        if not self._latex_expression:
            self._latex_expression = self.latex_numerator + '/ \\,\\Sigma'
        return self._latex_expression

    @property
    def latex_name(self):
        if not self._latex_name:
            self._latex_name = self.name
        return self._latex_name

    @property
    def percentage(self):
        self._percentage = (self.value / self.parent.value) * 100
        return self._percentage
<commit_msg>Made str_expression in ccobjects a property so that slowdown during do_symca is deferred to when object is needed.<commit_after>import numpy as np
from numpy import min, max, float, array, float64
#from PyscesToolBox import PyscesToolBox as PYCtools
from sympy import Symbol
#from IPython.display import Latex
from ...utils.misc import silence_print
from ...utils.plotting import Data2D
from ... import modeltools
from pysces import ModelMap

def cctype(obj):
    return 'ccobjects' in str(type(obj))

@silence_print
def get_state(mod, do_state=False):
    if do_state:
        mod.doState()
    ss = [getattr(mod, 'J_' + r) for r in mod.reactions] + \
        [getattr(mod, s + '_ss') for s in mod.species]
    return ss

@silence_print
def silent_mca(mod):
    mod.doMca()

def get_value_eval(expression,subs_dict):
    for k,v in subs_dict.iteritems():
        subs_dict[k] = float64(v)
    ans = eval(expression,{},subs_dict)
    return ans

class StateKeeper:
    def __init__(self, state):
        self._last_state_for_mca = state

    def do_mca_state(self, mod, state):
        if state != self._last_state_for_mca:
            silent_mca(mod)
            self._last_state_for_mca = state

class CCBase(object):

    """The base object for the control coefficients and control patterns"""

    def __init__(self, mod, name, expression, ltxe, state_keeper):
        super(CCBase, self).__init__()

        self.expression = expression
        self.mod = mod
        self._ltxe = ltxe
        self._state_keeper = state_keeper
        self.name = name
        self._latex_name = '\\Sigma'

        self._str_expression_ = None
        self._value = None
        self._latex_expression = None
        self._state_ = get_state(mod)

    @property
    def latex_expression(self):
        if not self._latex_expression:
            self._latex_expression = self._ltxe.expression_to_latex(
                self.expression
            )
        return self._latex_expression

    @property
    def latex_name(self):
        return self._latex_name

    @property
    def _str_expression(self):
        if not self._str_expression_:
            self._str_expression_ = str(self.expression)
        return self._str_expression_

    @property
    def value(self):
        """The value property. Calls self._calc_value() when self._value
        is None and returns self._value"""
        new_ss = get_state(self.mod)
        state_changed = new_ss != self._state_
        if state_changed:
            self._state_keeper.do_mca_state(self.mod, new_ss)
            self._calc_value()
            self._state_ = new_ss
        elif not self._value:
            self._calc_value()
        return self._value

    def _repr_latex_(self):
        return '$%s = %s = %.3f$' % (self.latex_name,
                                     self.latex_expression,
                                     self.value)

    def _calc_value(self):
        """Calculates the value of the expression"""
        keys = self.expression.atoms(Symbol)
        subsdict = {}
        for key in keys:
            str_key = str(key)
            subsdict[str_key] = getattr(self.mod, str_key)
        self._value = get_value_eval(self._str_expression,subsdict)

    def __repr__(self):
        return self.expression.__repr__()

    def __add__(self, other):
        if cctype(other):
            return self.expression.__add__(other.expression)
        else:
            return self.expression.__add__(other)

    def __mul__(self, other):
        if cctype(other):
            return self.expression.__mul__(other.expression)
        else:
            return self.expression.__mul__(other)

    def __div__(self, other):
        if cctype(other):
            return self.expression.__div__(other.expression)
        else:
            return self.expression.__div__(other)

    def __pow__(self, other):
        if cctype(other):
            return self.expression.__pow__(other.expression)
        else:
            return self.expression.__pow__(other)

class CCoef(CCBase):

    """The object the stores control coefficients. Inherits from CCBase"""

    def __init__(self, mod, name, expression, denominator, ltxe, state_keeper):
        super(CCoef, self).__init__(mod, name, expression, ltxe, state_keeper)
        self.numerator = expression
        self.denominator = denominator.expression
        self.expression = self.numerator / denominator.expression
        self.denominator_object = denominator

        self._latex_numerator = None
        self._latex_expression_full = None
        self._latex_expression = None
        self._latex_name = None

        self.control_patterns = None

        self._set_control_patterns()

    @property
    def latex_numerator(self):
        if not self._latex_numerator:
            self._latex_numerator = self._ltxe.expression_to_latex(
                self.numerator
            )
        return self._latex_numerator

    @property
    def latex_expression_full(self):
        if not self._latex_expression_full:
            full_expr = '\\frac{' + self.latex_numerator + '}{' \
                + self.denominator_object.latex_expression + '}'
            self._latex_expression_full = full_expr
        return self._latex_expression_full

    @property
    def latex_expression(self):
        if not self._latex_expression:
            self._latex_expression = '(' + self.latex_numerator + ')' + '/ \\,\\Sigma'
        return self._latex_expression

    @property
    def latex_name(self):
        if not self._latex_name:
            self._latex_name = self._ltxe.expression_to_latex(
                self.name
            )
        return self._latex_name

    #@property
    # def control_patterns(self):
    #    if not self._control_patterns:
    #        self._set_control_patterns()
    #    return self._control_patterns

    def parscan(self, parameter, scan_range, scan_type='percentage', init_return=False):
        """Performs a parameter scan and returns numpy array object
           with the parameter values in the first column and
           percentage contribution of each control pattern
           in subsequent columns

           Arguments:
           parameter   --  the parameter of the model to scan
           scan_range  --  the range across which to scan 'parameter'

           calls self._recalculate_value() for each value of
           parameter in scan_range"""

        assert scan_type in ['percentage', 'value']
        if scan_type is 'percentage':
            scan_res = [list() for i in range(len(self.control_patterns) + 1)]
        elif scan_type is 'value':
            scan_res = [list() for i in range(len(self.control_patterns) + 2)]
        scan_res[0] = scan_range

        # print type(scan_res)
        init = getattr(self.mod, parameter)
        for parvalue in scan_range:
            setattr(self.mod, parameter, parvalue)
            self.mod.SetQuiet()
            self.mod.doMca()
            self.mod.SetLoud()

            for i, cp in enumerate(self.control_patterns):
                # print type(scan_res[i+1])
                if scan_type is 'percentage':
                    scan_res[i + 1].append(cp.percentage)
                elif scan_type is 'value':
                    scan_res[i + 1].append(cp.value)

            if scan_type is 'value':
                scan_res[i + 2].append(self.value)

        if init_return:
            self.mod.SetQuiet()
            setattr(self.mod, parameter, init)
            self.mod.doMca()
            self.mod.SetLoud()

        if scan_type is 'value':
            column_names = [parameter] + [cp.name for cp in self.control_patterns] + [self.name]
            y_label = 'Control coefficient/pattern value'
        if scan_type is 'percentage':
            column_names = [parameter] + [cp.name for cp in self.control_patterns]
            y_label = 'Percentage Contribution'

        mm = ModelMap(self.mod)
        species = mm.hasSpecies()
        if parameter in species:
            x_label = '[%s]' % parameter.replace('_', ' ')
        else:
            x_label = parameter
        ax_properties = {'ylabel':y_label,
                         'xlabel':x_label,
                         'xscale': 'linear',
                         'yscale': 'linear',
                         'xlim': [scan_range[0],scan_range[-1]]}
        data_array = array(scan_res, dtype=np.float).transpose()
        data = Data2D(self.mod, column_names, data_array, self._ltxe, 'symca', ax_properties)
        return data

    def _recalculate_value(self):
        """Recalculates the control coefficients and control pattern
           values. calls _calc_value() for self and each control
           pattern. Useful for when model parameters change"""
        keys = self.expression.atoms(Symbol)
        subsdict = {}
        for key in keys:
            str_key = str(key)
            subsdict[str_key] = getattr(self.mod, str_key)
        for pattern in self.control_patterns:
            pattern._calc_value(subsdict)
        self._calc_value()

    def _calc_value(self):
        """Calculates the numeric value of the control pattern from the
           values of its control patterns."""
        self._value = sum([pattern.value for pattern in self.control_patterns])

    def _set_control_patterns(self):
        """Divides control coefficient into control patterns and saves
           results in self.CPx where x is a number is the number of the
           control pattern as it appears in in control coefficient
           expression"""
        patterns = self.numerator.as_coeff_add()[1]
        cps = []
        for i, pattern in enumerate(patterns):
            name = 'CP' + str(1 + i)
            cp = CPattern(self.mod,
                          name,
                          pattern,
                          self.denominator_object,
                          self,
                          self._ltxe,
                          self._state_keeper)
            setattr(self, name, cp)
            cps.append(cp)
        self.control_patterns = cps
        #assert self._check_control_patterns == True

    def _check_control_patterns(self):
        """Checks that all control patterns are either positive or negative"""
        all_same = False
        poscomp = [i.value > 0 for i in self.control_patterns]
        negcomp = [i.value < 0 for i in self.control_patterns]
        if all(poscomp):
            all_same = True
        elif all(negcomp):
            all_same = True
        return all_same

class CPattern(CCBase):

    """docstring for CPattern"""

    def __init__(self,
                 mod,
                 name,
                 expression,
                 denominator,
                 parent,
                 ltxe,
                 state_keeper):
        super(CPattern, self).__init__(mod,
                                       name,
                                       expression,
                                       ltxe,
                                       state_keeper)
        self.numerator = expression
        self.denominator = denominator.expression
        self.expression = self.numerator / denominator.expression
        self.denominator_object = denominator
        self.parent = parent

        self._latex_numerator = None
        self._latex_expression_full = None
        self._latex_expression = None
        self._latex_name = None
        self._percentage = None

    def _calc_value(self,subsdict=None):
        """Calculates the value of the expression"""
        if subsdict is None:
            keys = self.expression.atoms(Symbol)
            subsdict = {}
            for key in keys:
                str_key = str(key)
                subsdict[str_key] = getattr(self.mod, str_key)
        self._value = get_value_eval(self._str_expression,subsdict)

    @property
    def latex_numerator(self):
        if not self._latex_numerator:
            self._latex_numerator = self._ltxe.expression_to_latex(
                self.numerator
            )
        return self._latex_numerator

    @property
    def latex_expression_full(self):
        if not self._latex_expression_full:
            full_expr = '\\frac{' + self.latex_numerator + '}{' \
                + self.denominator_object.latex_expression + '}'
            self._latex_expression_full = full_expr
        return self._latex_expression_full

    @property
    def latex_expression(self):
        if not self._latex_expression:
            self._latex_expression = self.latex_numerator + '/ \\,\\Sigma'
        return self._latex_expression

    @property
    def latex_name(self):
        if not self._latex_name:
            self._latex_name = self.name
        return self._latex_name

    @property
    def percentage(self):
        self._percentage = (self.value / self.parent.value) * 100
        return self._percentage
<|endoftext|>


<commit_before>from entrypoint2 import entrypoint
from psidialogs.backendloader import BackendLoader
import inspect
import logging
import psidialogs

log = logging.getLogger(__name__)

@entrypoint
def open(backend, func, title='', message='', choices='', text=''):
    BackendLoader().force(backend)

    args=dict(
              title=title,
              message=message,
              choices=choices.split(','),
              text=text,
              )

    funcs = psidialogs.FUNCTIONS
    log.debug('functions found:')
    log.debug(funcs)
    log.debug('searching for:')
    log.debug(func)
    f = None
    for x in funcs:
        if x.__name__ == func:
            f = x
    assert f
    argnames, varargs, varkw, defaults = inspect.getargspec(f)
    #argnames = psidialogs.argnames(func)

    args = dict([(k, v) for (k, v) in args.items() if k in argnames])
    result = None
    exec 'result = psidialogs.%s(**args)' % (func)
    log.debug('result:' + str(result))
    print result

<commit_msg>examples<commit_after>from entrypoint2 import entrypoint
from psidialogs.backendloader import BackendLoader
import inspect
import logging
import psidialogs

log = logging.getLogger(__name__)

@entrypoint
def opendialog(backend, func, title='', message='', choices='', text=''):
    BackendLoader().force(backend)

    args=dict(
              title=title,
              message=message,
              choices=choices.split(','),
              text=text,
              )

    funcs = psidialogs.FUNCTIONS
    log.debug('functions found:')
    log.debug(funcs)
    log.debug('searching for:')
    log.debug(func)
    f = None
    for x in funcs:
        if x.__name__ == func:
            f = x
    assert f
    argnames, varargs, varkw, defaults = inspect.getargspec(f)

    args = dict([(k, v) for (k, v) in args.items() if k in argnames])
    result = None
    exec 'result = psidialogs.%s(**args)' % (func)
    log.debug('result:' + str(result))
    print result

<|endoftext|>


<commit_before>import json
from django.core import serializers
import re

def format_object_to_plist(obj):
    """
    Formats all keys from a python-like dictionary to plist-like variable
    """
    dictionary = json.loads(serializers.serialize('json', [obj]))
    dictionary = dictionary[0]['fields']
    return {(re.sub(r"_([a-z0-9])", underscore_to_camel_case, str(key))).capitalize():  value for key, value in dictionary.items()}

def underscore_to_camel_case(match):
    """
    Transforms python-like variable (with _ separator) to plist-like variable (camel-cased)
    """
    return match.group(1).upper()
<commit_msg>Propper format of variable from underscored to camel cased + capitalized<commit_after>import json
from django.core import serializers
import re

def format_object_to_plist(obj):
    """
    Formats all keys from a python-like dictionary to plist-like variable
    """
    dictionary = json.loads(serializers.serialize('json', [obj]))
    dictionary = dictionary[0]['fields']
    result = {}
    for key, value in dictionary.items():
        # The goal here is to transform variable looking like this
        #   this_is_1_random_var
        # to this
        #   ThisIs1RandomVar
        # So next line replaces ever _x with X
        key = re.sub(r"_([a-z0-9])", underscore_to_camel_case, str(key))
        # And this lines replaces the first letter with it's upper version
        key = key[:1].capitalize() + key[1:]
        result[key] = value
    return result

def underscore_to_camel_case(match):
    """
    Transforms python-like variable (with _ separator) to plist-like variable (camel-cased)
    """
    return match.group(1).upper()
<|endoftext|>