<commit_before>import os
import requests
from time import time
from Levenshtein import ratio
from collections import defaultdict

from webpage import PublisherWebpage, WebpageInOpenRepo, WebpageInUnknownRepo
from util import elapsed
from util import normalize

DEBUG_BASE = False

def get_urls_from_our_base_doc(doc):
    response = []

    if "urls" in doc:
        # pmc can only add pmc urls.  otherwise has junk about dois that aren't actually open.
        if u"PubMed Central (PMC)" in doc["sources"]:
            for url in doc["urls"]:
                if "/pmc/" in url and url != "http://www.ncbi.nlm.nih.gov/pmc/articles/PMC":
                    response += [url]
        else:
            response += doc["urls"]

    # oxford IR doesn't return URLS, instead it returns IDs from which we can build URLs
    # example: https://www.base-search.net/Record/5c1cf4038958134de9700b6144ae6ff9e78df91d3f8bbf7902cb3066512f6443/
    if "sources" in doc and "Oxford University Research Archive (ORA)" in doc["sources"]:
        if "relations" in doc:
            for relation in doc["relations"]:
                if relation.startswith("uuid"):
                    response += [u"https://ora.ox.ac.uk/objects/{}".format(relation)]

    return response

def normalize_title_for_querying(title):
    if not title:
        return ""
    title_to_query = ""
    title = title.lower()
    # can't just replace all punctuation because ' replaced with ? gets no hits
    title = title.replace('"', "?")
    title = title.replace('#', "?")
    title = title.replace('=', "?")
    title = title.replace('&', "?")
    title = title.replace('%', "?")
    title = title.replace('/', "?")
    title = title.replace('!', "?")
    # title = title.replace('-', "*")

    # only bother looking up titles that are at least 3 words long
    title_words = title.split()
    if len(title_words) >= 3:
        # only look up the first 12 words
        title_to_query = u" ".join(title_words[0:12])

    return title_to_query

def call_our_base(my_pub):
    if not my_pub:
        return

    webpages_to_return = []
    found_a_base1 = False
    title = my_pub.best_title
    title_to_query = normalize_title_for_querying(title)

    if not title_to_query:
        return webpages_to_return

    # now do the lookup in base
    # query_string = u'title=%22{}%22'.format(title_to_query)
    query_string = u'title:({})'.format(title_to_query)

    if my_pub.first_author_lastname:
        query_string += u" AND authors:({} OR {})".format(
            my_pub.first_author_lastname, normalize(my_pub.first_author_lastname))

    # if my_product.doi:
    #     query_string += u" OR urls={}".format(my_product.doi)

    url_template = u"{base_url}/base/_search?pretty&size=20&q={query_string}"
    url = url_template.format(base_url=os.getenv("BASE_URL"), query_string=query_string)

    if DEBUG_BASE:
        print u"calling our base with {}\n".format(url)

    start_time = time()
    r = None
    try:
        r = requests.get(url, timeout=10)
        # print u"** querying with {} titles took {}s".format(len(titles), elapsed(start_time))
    except requests.exceptions.ConnectionError:
        print u"connection error in call_our_base, skipping."
    except requests.Timeout:
        print u"TIMEOUT error in call_our_base, skipping."

    if r != None and r.status_code != 200:
        print u"problem searching base! status_code={}".format(r.status_code)
        my_pub.base_dcoa = u"base query error: status_code={}".format(r.status_code)

    else:
        try:
            data = r.json()["hits"]["hits"]

            for hit in data:
                doc = hit["_source"]
                urls_for_this_hit = get_urls_from_our_base_doc(doc)
                if not urls_for_this_hit:
                    continue

                title_matches = False
                normalized_pub_title = normalize(my_pub.best_title)
                normalized_base_title = normalize(doc["title"])

                lev_ratio = ratio(normalized_pub_title, normalized_base_title)

                if len(my_pub.best_title) < 40 or len(doc["title"]) < 40:
                    if normalized_pub_title==normalized_base_title:
                        title_matches = True
                        if DEBUG_BASE:
                            print u"exact match on short titles", urls_for_this_hit
                else:
                    if normalized_pub_title in normalized_base_title:
                        title_matches = True
                        if DEBUG_BASE:
                            print u"subset title match on ", urls_for_this_hit
                    if normalized_base_title in normalized_pub_title:
                        title_matches = True
                        if DEBUG_BASE:
                            print u"subset title match on", urls_for_this_hit

                # only fuzzy match if we don't have exact matches
                # if doing a fuzzy match, make sure the query included a last name
                if not title_matches:
                    # if DEBUG_BASE:
                    #     print u"{}\n{}\n{}\n{}".format(lev_ratio, normalized_pub_title, normalized_base_title, get_urls_from_our_base_doc(doc))

                    if my_pub.first_author_lastname:
                        if lev_ratio > 0.8:
                            title_matches = True
                            if DEBUG_BASE:
                                print u"no last name in publication, match by lev distance"
                    else:
                        if lev_ratio > 0.95:
                            title_matches = True
                            if DEBUG_BASE:
                                print u"HAS last name in publication, match by lev distance"

                if title_matches:
                    if doc["oa"] == 1:
                        found_a_base1 = True
                        for url in get_urls_from_our_base_doc(doc):
                            my_webpage = WebpageInOpenRepo(url=url, related_pub=my_pub)
                            if "rights" in doc:
                                my_webpage.scraped_license = oa_local.find_normalized_license(format(doc["rights"]))
                            webpages_to_return.append(my_webpage)

                    elif doc["oa"] == 2 and not found_a_base1:
                        for url in get_urls_from_our_base_doc(doc):
                            my_webpage = WebpageInUnknownRepo(url=url, related_pub=my_pub)
                            webpages_to_return.append(my_webpage)

        except ValueError:  # includes simplejson.decoder.JSONDecodeError
            print u'decoding JSON has failed base response'
            my_pub.base_dcoa = u"base lookup error: json response parsing"
        except AttributeError:  # no json
            # print u"no hit with title {}".format(doc["dctitle"])
            # print u"normalized: {}".format(normalize(doc["dctitle"]))
            pass

    print u"finished base step of set_fulltext_urls with in {}s".format(
        elapsed(start_time, 2))

    if DEBUG_BASE:
        print u"found these webpages in base: {}".format(webpages_to_return)
    return webpages_to_return

<commit_msg>printouts for base speed<commit_after>import os
import requests
from time import time
from Levenshtein import ratio
from collections import defaultdict

from webpage import PublisherWebpage, WebpageInOpenRepo, WebpageInUnknownRepo
from util import elapsed
from util import normalize

DEBUG_BASE = False

def get_urls_from_our_base_doc(doc):
    response = []

    if "urls" in doc:
        # pmc can only add pmc urls.  otherwise has junk about dois that aren't actually open.
        if u"PubMed Central (PMC)" in doc["sources"]:
            for url in doc["urls"]:
                if "/pmc/" in url and url != "http://www.ncbi.nlm.nih.gov/pmc/articles/PMC":
                    response += [url]
        else:
            response += doc["urls"]

    # oxford IR doesn't return URLS, instead it returns IDs from which we can build URLs
    # example: https://www.base-search.net/Record/5c1cf4038958134de9700b6144ae6ff9e78df91d3f8bbf7902cb3066512f6443/
    if "sources" in doc and "Oxford University Research Archive (ORA)" in doc["sources"]:
        if "relations" in doc:
            for relation in doc["relations"]:
                if relation.startswith("uuid"):
                    response += [u"https://ora.ox.ac.uk/objects/{}".format(relation)]

    return response

def normalize_title_for_querying(title):
    if not title:
        return ""
    title_to_query = ""
    title = title.lower()
    # can't just replace all punctuation because ' replaced with ? gets no hits
    title = title.replace('"', "?")
    title = title.replace('#', "?")
    title = title.replace('=', "?")
    title = title.replace('&', "?")
    title = title.replace('%', "?")
    title = title.replace('/', "?")
    title = title.replace('!', "?")
    # title = title.replace('-', "*")

    # only bother looking up titles that are at least 3 words long
    title_words = title.split()
    if len(title_words) >= 3:
        # only look up the first 12 words
        title_to_query = u" ".join(title_words[0:12])

    return title_to_query

def call_our_base(my_pub):
    if not my_pub:
        return

    webpages_to_return = []
    found_a_base1 = False
    title = my_pub.best_title
    title_to_query = normalize_title_for_querying(title)

    if not title_to_query:
        return webpages_to_return

    # now do the lookup in base
    # query_string = u'title=%22{}%22'.format(title_to_query)
    query_string = u'title:({})'.format(title_to_query)

    if my_pub.first_author_lastname:
        query_string += u" AND authors:({} OR {})".format(
            my_pub.first_author_lastname, normalize(my_pub.first_author_lastname))

    # if my_product.doi:
    #     query_string += u" OR urls={}".format(my_product.doi)

    url_template = u"{base_url}/base/_search?pretty&size=20&q={query_string}"
    url = url_template.format(base_url=os.getenv("BASE_URL"), query_string=query_string)

    if DEBUG_BASE:
        print u"calling our base with {}\n".format(url)

    print u"calling our base with {}\n".format(url)

    start_time = time()
    r = None
    try:
        r = requests.get(url, timeout=10)
        # print u"** querying with {} titles took {}s".format(len(titles), elapsed(start_time))
    except requests.exceptions.ConnectionError:
        print u"connection error in call_our_base, skipping."
    except requests.Timeout:
        print u"TIMEOUT error in call_our_base, skipping."

    if r != None and r.status_code != 200:
        print u"problem searching base! status_code={}".format(r.status_code)
        my_pub.base_dcoa = u"base query error: status_code={}".format(r.status_code)

    else:
        try:
            data = r.json()["hits"]["hits"]

            for hit in data:
                doc = hit["_source"]
                urls_for_this_hit = get_urls_from_our_base_doc(doc)
                if not urls_for_this_hit:
                    continue

                title_matches = False
                normalized_pub_title = normalize(my_pub.best_title)
                normalized_base_title = normalize(doc["title"])

                lev_ratio = ratio(normalized_pub_title, normalized_base_title)

                if len(my_pub.best_title) < 40 or len(doc["title"]) < 40:
                    if normalized_pub_title==normalized_base_title:
                        title_matches = True
                        if DEBUG_BASE:
                            print u"exact match on short titles", urls_for_this_hit
                else:
                    if normalized_pub_title in normalized_base_title:
                        title_matches = True
                        if DEBUG_BASE:
                            print u"subset title match on ", urls_for_this_hit
                    if normalized_base_title in normalized_pub_title:
                        title_matches = True
                        if DEBUG_BASE:
                            print u"subset title match on", urls_for_this_hit

                # only fuzzy match if we don't have exact matches
                # if doing a fuzzy match, make sure the query included a last name
                if not title_matches:
                    # if DEBUG_BASE:
                    #     print u"{}\n{}\n{}\n{}".format(lev_ratio, normalized_pub_title, normalized_base_title, get_urls_from_our_base_doc(doc))

                    if my_pub.first_author_lastname:
                        if lev_ratio > 0.8:
                            title_matches = True
                            if DEBUG_BASE:
                                print u"no last name in publication, match by lev distance"
                    else:
                        if lev_ratio > 0.95:
                            title_matches = True
                            if DEBUG_BASE:
                                print u"HAS last name in publication, match by lev distance"

                if title_matches:
                    if doc["oa"] == 1:
                        found_a_base1 = True
                        for url in get_urls_from_our_base_doc(doc):
                            my_webpage = WebpageInOpenRepo(url=url, related_pub=my_pub)
                            if "rights" in doc:
                                my_webpage.scraped_license = oa_local.find_normalized_license(format(doc["rights"]))
                            webpages_to_return.append(my_webpage)

                    elif doc["oa"] == 2 and not found_a_base1:
                        for url in get_urls_from_our_base_doc(doc):
                            my_webpage = WebpageInUnknownRepo(url=url, related_pub=my_pub)
                            webpages_to_return.append(my_webpage)

        except ValueError:  # includes simplejson.decoder.JSONDecodeError
            print u'decoding JSON has failed base response'
            my_pub.base_dcoa = u"base lookup error: json response parsing"
        except AttributeError:  # no json
            # print u"no hit with title {}".format(doc["dctitle"])
            # print u"normalized: {}".format(normalize(doc["dctitle"]))
            pass

    print u"finished base step of set_fulltext_urls with in {}s".format(
        elapsed(start_time, 2))

    if DEBUG_BASE:
        print u"found these webpages in base: {}".format(webpages_to_return)
    return webpages_to_return

<|endoftext|>


<commit_before># Django settings for thedp project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': 'db.sqlite',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'test!kskf-4=)f_tbn$scxdfmg%17wetw@1iu%$=0sy0k@33oh)x9e'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'testproject.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'testproject.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',

    'armstrong.core.arm_layout',
    'thedp',
]

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
<commit_msg>add django humanize to installed apps<commit_after># Django settings for thedp project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': 'db.sqlite',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'test!kskf-4=)f_tbn$scxdfmg%17wetw@1iu%$=0sy0k@33oh)x9e'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'testproject.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'testproject.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'django.contrib.humanize',

    'armstrong.core.arm_layout',
    'thedp',
]

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Generate notes from an ngram model
USAGE: ./ngram.py MODELFILE NUM_NOTES
"""

import numpy as np
from collections import deque
import sys,random,pdb,os,music21
from ngram_helper import pitch_to_str
from audiolazy import midi2str
from optparse import OptionParser

TWELVE_BAR_BLUES = [1,1,1,1,4,4,1,1,5,4,1,1]
DURKS_PER_MEASURE = 32

class NgramGenerator():
    def __init__(self,key,chord_progression,model_dir):
        """
        @param key: music21 key. equal to key model was trained in
        @param chord_progression: series of music21 roman numerals
        @param model_dir: directory containing the 7 separate trained models
        @return:
        """
        self.durk = 0
        self.key = key
        self.chord_progression = chord_progression
        self.last_two = deque()
        self.rn = None

        # load the various rn models
        self.counts = []
        for i in xrange(7):
            with open(os.path.join(model_dir,str(i+1) + ".npy")) as f:
                self.counts.append(np.load(f))

    def __iter__(self):
        return self

    def _choose_note_once(self,rn,last_two):
        mat = self.counts[rn.scaleDegree-1]

        row = mat[tuple(last_two)]
        total = np.sum(row)

        rand = (random.randint(0,total-1) if total > 0 else 0)

        i = 0
        rand -= row[i]

        while rand > 0:
            i += 1
            rand -= row[i]

        return music21.pitch.Pitch(midi2str(i))

    def choose_note(self,rn,last_two,depth=0):
        # failed too many times. just play the root of the scale
        if depth >= sys.getrecursionlimit() * 0.75:
            return self.key.getScale().chord.root()

        # try generating a pitch.
        gen = self._choose_note_once(rn,last_two)
        for pitch in rn.pitches:
            # ensure the pitch is in the rn
            if gen.pitchClass == pitch.pitchClass:
                return gen

        # failed. try again to generate
        return self.choose_note(rn,last_two,depth+1)

    # convert note to 1->7 for GA
    def calc_note_val(self,pitch_class):
        pitches = self.key.getScale().getChord().pitches

        for i in xrange(7):
            if pitches[i].pitchClass == pitch_class:
                return i+1

        # should never happen!
        return 1

    def next(self):

        if self.durk < len(self.chord_progression) * DURKS_PER_MEASURE:

            rn = self.chord_progression[self.durk / DURKS_PER_MEASURE]

            if rn != self.rn:
                self.last_two = deque()

                # append the first, third
                scale = self.key.getScale()
                self.last_two.append(scale.chord.root().midi)
                self.last_two.append(scale.chord.third.midi)

            self.rn = rn
            pitch = self.choose_note(rn,self.last_two)

            self.last_two.append(pitch.midi)
            self.last_two.popleft()

            self.durk += 4

            print self.rn, pitch
            return self.calc_note_val(pitch.pitchClass)
        else:
            raise StopIteration()

def construct_roman_numerals(key,int_seq):
    rns = []
    for i in int_seq:
        rns.append(music21.roman.RomanNumeral(i,key))
    return rns

def generate(options,cp):
    key = music21.key.Key(options.key)
    cp = construct_roman_numerals(key,cp)
    generator = NgramGenerator(key,cp,options.model_dir)

    return [note for note in generator]

# if __name__ == '__main__':
parser = OptionParser()

parser.add_option("-m", "--model-dir", dest="model_dir")
parser.add_option("-k", "--key", dest="key", default="C")

(options, args) = parser.parse_args()

print map(pitch_to_str,generate(options,TWELVE_BAR_BLUES))<commit_msg>better printing<commit_after>#!/usr/bin/env python

"""
Generate notes from an ngram model
USAGE: ./ngram.py MODELFILE NUM_NOTES
"""

import numpy as np
from collections import deque
import sys,random,pdb,os,music21
from ngram_helper import pitch_to_str
from audiolazy import midi2str
from optparse import OptionParser

TWELVE_BAR_BLUES = [1,1,1,1,4,4,1,1,5,4,1,1]
DURKS_PER_MEASURE = 32

class NgramGenerator():
    def __init__(self,key,chord_progression,model_dir):
        """
        @param key: music21 key. equal to key model was trained in
        @param chord_progression: series of music21 roman numerals
        @param model_dir: directory containing the 7 separate trained models
        @return:
        """
        self.durk = 0
        self.key = key
        self.chord_progression = chord_progression
        self.last_two = deque()
        self.rn = None

        # load the various rn models
        self.counts = []
        for i in xrange(7):
            with open(os.path.join(model_dir,str(i+1) + ".npy")) as f:
                self.counts.append(np.load(f))

    def __iter__(self):
        return self

    def _choose_note_once(self,rn,last_two):
        mat = self.counts[rn.scaleDegree-1]

        row = mat[tuple(last_two)]
        total = np.sum(row)

        rand = (random.randint(0,total-1) if total > 0 else 0)

        i = 0
        rand -= row[i]

        while rand > 0:
            i += 1
            rand -= row[i]

        return music21.pitch.Pitch(midi2str(i))

    def choose_note(self,rn,last_two,depth=0):
        # failed too many times. just play the root of the scale
        if depth >= sys.getrecursionlimit() * 0.75:
            return self.key.getScale().chord.root()

        # try generating a pitch.
        gen = self._choose_note_once(rn,last_two)
        for pitch in rn.pitches:
            # ensure the pitch is in the rn
            if gen.pitchClass == pitch.pitchClass:
                return gen

        # failed. try again to generate
        return self.choose_note(rn,last_two,depth+1)

    # convert note to 1->7 for GA
    def calc_note_val(self,pitch_class):
        pitches = self.key.getScale().getChord().pitches

        for i in xrange(7):
            if pitches[i].pitchClass == pitch_class:
                return i+1

        # should never happen!
        return 1

    def next(self):

        if self.durk < len(self.chord_progression) * DURKS_PER_MEASURE:

            rn = self.chord_progression[self.durk / DURKS_PER_MEASURE]

            if rn != self.rn:
                self.last_two = deque()

                # append the first, third
                scale = self.key.getScale()
                self.last_two.append(scale.chord.root().midi)
                self.last_two.append(scale.chord.third.midi)

            self.rn = rn
            pitch = self.choose_note(rn,self.last_two)

            # update last_two
            self.last_two.append(pitch.midi)
            self.last_two.popleft()

            self.durk += 4

            # calculate 1->7 value
            val = self.calc_note_val(pitch.pitchClass)
            print self.key, self.rn.figure, pitch, val

            return val
        else:
            raise StopIteration()

def construct_roman_numerals(key,int_seq):
    rns = []
    for i in int_seq:
        rns.append(music21.roman.RomanNumeral(i,key))
    return rns

def generate(options,cp):
    key = music21.key.Key(options.key)
    cp = construct_roman_numerals(key,cp)
    generator = NgramGenerator(key,cp,options.model_dir)

    return [note for note in generator]

# if __name__ == '__main__':
parser = OptionParser()

parser.add_option("-m", "--model-dir", dest="model_dir")
parser.add_option("-k", "--key", dest="key", default="C")

(options, args) = parser.parse_args()

print map(pitch_to_str,generate(options,TWELVE_BAR_BLUES))<|endoftext|>


<commit_before><commit_msg>[FIX] custom_partner_it: Añadido que al ser un cliente potencial permita guardarlo sin PEC<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
httpbin.helpers
~~~~~~~~~~~~~~~

This module provides helper functions for httpbin.
"""

import json
import base64
from hashlib import md5, sha256
from werkzeug.http import parse_authorization_header

from flask import request, make_response
from six.moves.urllib.parse import urlparse, urlunparse

from .structures import CaseInsensitiveDict

ASCII_ART = """
    -=[ teapot ]=-

       _...._
     .'  _ _ `.
    | ."` ^ `". _,
    \_;`"---"`|//
      |       ;/
      \_     _/
        `\"\"\"`
"""

REDIRECT_LOCATION = '/redirect/1'

ENV_HEADERS = (
    'X-Varnish',
    'X-Request-Start',
    'X-Heroku-Queue-Depth',
    'X-Real-Ip',
    'X-Forwarded-Proto',
    'X-Forwarded-Protocol',
    'X-Forwarded-Ssl',
    'X-Heroku-Queue-Wait-Time',
    'X-Forwarded-For',
    'X-Heroku-Dynos-In-Use',
    'X-Forwarded-For',
    'X-Forwarded-Protocol',
    'X-Forwarded-Port',
    'Runscope-Service'
)

ROBOT_TXT = """User-agent: *
Disallow: /deny
"""

ACCEPTED_MEDIA_TYPES = [
    'image/webp',
    'image/svg+xml',
    'image/jpeg',
    'image/png',
    'image/*'
]

ANGRY_ASCII ="""
          .-''''''-.
        .' _      _ '.
       /   O      O   \\
      :                :
      |                |
      :       __       :
       \  .-"`  `"-.  /
        '.          .'
          '-......-'
     YOU SHOULDN'T BE HERE
"""

def json_safe(string, content_type='application/octet-stream'):
    """Returns JSON-safe version of `string`.

    If `string` is a Unicode string or a valid UTF-8, it is returned unmodified,
    as it can safely be encoded to JSON string.

    If `string` contains raw/binary data, it is Base64-encoded, formatted and
    returned according to "data" URL scheme (RFC2397). Since JSON is not
    suitable for binary data, some additional encoding was necessary; "data"
    URL scheme was chosen for its simplicity.
    """
    try:
        string = string.decode('utf-8')
        json.dumps(string)
        return string
    except (ValueError, TypeError):
        return b''.join([
            b'data:',
            content_type.encode('utf-8'),
            b';base64,',
            base64.b64encode(string)
        ]).decode('utf-8')

def get_files():
    """Returns files dict from request context."""

    files = dict()

    for k, v in request.files.items():
        content_type = request.files[k].content_type or 'application/octet-stream'
        val = json_safe(v.read(), content_type)
        if files.get(k):
            if not isinstance(files[k], list):
                files[k] = [files[k]]
            files[k].append(val)
        else:
            files[k] = val

    return files

def get_headers(hide_env=True):
    """Returns headers dict from request context."""

    headers = dict(request.headers.items())

    if hide_env and ('show_env' not in request.args):
        for key in ENV_HEADERS:
            try:
                del headers[key]
            except KeyError:
                pass

    return CaseInsensitiveDict(headers.items())

def semiflatten(multi):
    """Convert a MutiDict into a regular dict. If there are more than one value
    for a key, the result will have a list of values for the key. Otherwise it
    will have the plain value."""
    if multi:
        result = multi.to_dict(flat=False)
        for k, v in result.items():
            if len(v) == 1:
                result[k] = v[0]
        return result
    else:
        return multi

def get_url(request):
    """
    Since we might be hosted behind a proxy, we need to check the
    X-Forwarded-Proto, X-Forwarded-Protocol, or X-Forwarded-SSL headers
    to find out what protocol was used to access us.
    """
    protocol = request.headers.get('X-Forwarded-Proto') or request.headers.get('X-Forwarded-Protocol')
    if protocol is None and request.headers.get('X-Forwarded-Ssl') == 'on':
        protocol = 'https'
    if protocol is None:
        return request.url
    url = list(urlparse(request.url))
    url[0] = protocol
    return urlunparse(url)

def get_dict(*keys, **extras):
    """Returns request dict of given keys."""

    _keys = ('url', 'args', 'form', 'data', 'origin', 'headers', 'files', 'json')

    assert all(map(_keys.__contains__, keys))
    data = request.data
    form = semiflatten(request.form)

    try:
        _json = json.loads(data.decode('utf-8'))
    except (ValueError, TypeError):
        _json = None

    d = dict(
        url=get_url(request),
        args=semiflatten(request.args),
        form=form,
        data=json_safe(data),
        origin=request.headers.get('X-Forwarded-For', request.remote_addr),
        headers=get_headers(),
        files=get_files(),
        json=_json
    )

    out_d = dict()

    for key in keys:
        out_d[key] = d.get(key)

    out_d.update(extras)

    return out_d

def status_code(code):
    """Returns response object of given status code."""

    redirect = dict(headers=dict(location=REDIRECT_LOCATION))

    code_map = {
        301: redirect,
        302: redirect,
        303: redirect,
        304: dict(data=''),
        305: redirect,
        307: redirect,
        401: dict(headers={'WWW-Authenticate': 'Basic realm="Fake Realm"'}),
        402: dict(
            data='Fuck you, pay me!',
            headers={
                'x-more-info': 'http://vimeo.com/22053820'
            }
        ),
        406: dict(data=json.dumps({
                'message': 'Client did not request a supported media type.',
                'accept': ACCEPTED_MEDIA_TYPES
            }),
            headers={
                'Content-Type': 'application/json'
            }),
        407: dict(headers={'Proxy-Authenticate': 'Basic realm="Fake Realm"'}),
        418: dict(  # I'm a teapot!
            data=ASCII_ART,
            headers={
                'x-more-info': 'http://tools.ietf.org/html/rfc2324'
            }
        ),

    }

    r = make_response()
    r.status_code = code

    if code in code_map:

        m = code_map[code]

        if 'data' in m:
            r.data = m['data']
        if 'headers' in m:
            r.headers = m['headers']

    return r

def check_basic_auth(user, passwd):
    """Checks user authentication using HTTP Basic Auth."""

    auth = request.authorization
    return auth and auth.username == user and auth.password == passwd

# Digest auth helpers
# qop is a quality of protection

def H(data, algorithm):
    if algorithm == 'SHA-256':
        return sha256(data).hexdigest()
    else:
        return md5(data).hexdigest()

def HA1(realm, username, password, algorithm):
    """Create HA1 hash by realm, username, password

    HA1 = md5(A1) = MD5(username:realm:password)
    """
    if not realm:
        realm = u''
    return H(b":".join([username.encode('utf-8'),
                           realm.encode('utf-8'),
                           password.encode('utf-8')]), algorithm)

def HA2(credentails, request, algorithm):
    """Create HA2 md5 hash

    If the qop directive's value is "auth" or is unspecified, then HA2:
        HA2 = md5(A2) = MD5(method:digestURI)
    If the qop directive's value is "auth-int" , then HA2 is
        HA2 = md5(A2) = MD5(method:digestURI:MD5(entityBody))
    """
    if credentails.get("qop") == "auth" or credentails.get('qop') is None:
        return H(b":".join([request['method'].encode('utf-8'), request['uri'].encode('utf-8')]), algorithm)
    elif credentails.get("qop") == "auth-int":
        for k in 'method', 'uri', 'body':
            if k not in request:
                raise ValueError("%s required" % k)
        return H("%s:%s:%s" % (request['method'],
                               request['uri'],
                               H(request['body'], algorithm)), algorithm)
    raise ValueError

def response(credentails, password, request):
    """Compile digest auth response

    If the qop directive's value is "auth" or "auth-int" , then compute the response as follows:
       RESPONSE = MD5(HA1:nonce:nonceCount:clienNonce:qop:HA2)
    Else if the qop directive is unspecified, then compute the response as follows:
       RESPONSE = MD5(HA1:nonce:HA2)

    Arguments:
    - `credentails`: credentails dict
    - `password`: request user password
    - `request`: request dict
    """
    response = None
    algorithm = credentails.get('algorithm')
    HA1_value = HA1(
        credentails.get('realm'),
        credentails.get('username'),
        password,
        algorithm
    )
    HA2_value = HA2(credentails, request, algorithm)
    if credentails.get('qop') is None:
        response = H(b":".join([
            HA1_value.encode('utf-8'),
            credentails.get('nonce', '').encode('utf-8'),
            HA2_value.encode('utf-8')
        ]), algorithm)
    elif credentails.get('qop') == 'auth' or credentails.get('qop') == 'auth-int':
        for k in 'nonce', 'nc', 'cnonce', 'qop':
            if k not in credentails:
                raise ValueError("%s required for response H" % k)
        response = H(b":".join([HA1_value.encode('utf-8'),
                               credentails.get('nonce').encode('utf-8'),
                               credentails.get('nc').encode('utf-8'),
                               credentails.get('cnonce').encode('utf-8'),
                               credentails.get('qop').encode('utf-8'),
                               HA2_value.encode('utf-8')]), algorithm)
    else:
        raise ValueError("qop value are wrong")

    return response

def check_digest_auth(user, passwd):
    """Check user authentication using HTTP Digest auth"""

    if request.headers.get('Authorization'):
        credentails = parse_authorization_header(request.headers.get('Authorization'))
        if not credentails:
            return
        response_hash = response(credentails, passwd, dict(uri=request.script_root + request.path,
                                                           body=request.data,
                                                           method=request.method))
        if credentails.get('response') == response_hash:
            return True
    return False

def secure_cookie():
    """Return true if cookie should have secure attribute"""
    return request.environ['wsgi.url_scheme'] == 'https'

def __parse_request_range(range_header_text):
    """ Return a tuple describing the byte range requested in a GET request
    If the range is open ended on the left or right side, then a value of None
    will be set.
    RFC7233: http://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7233.html#header.range
    Examples:
      Range : bytes=1024-
      Range : bytes=10-20
      Range : bytes=-999
    """

    left = None
    right = None

    if not range_header_text:
        return left, right

    range_header_text = range_header_text.strip()
    if not range_header_text.startswith('bytes'):
        return left, right

    components = range_header_text.split("=")
    if len(components) != 2:
        return left, right

    components = components[1].split("-")

    try:
        right = int(components[1])
    except:
        pass

    try:
        left = int(components[0])
    except:
        pass

    return left, right

def get_request_range(request_headers, upper_bound):
    first_byte_pos, last_byte_pos = __parse_request_range(request_headers['range'])

    if first_byte_pos is None and last_byte_pos is None:
        # Request full range
        first_byte_pos = 0
        last_byte_pos = upper_bound - 1
    elif first_byte_pos is None:
        # Request the last X bytes
        first_byte_pos = max(0, upper_bound - last_byte_pos)
        last_byte_pos = upper_bound - 1
    elif last_byte_pos is None:
        # Request the last X bytes
        last_byte_pos = upper_bound - 1

    return first_byte_pos, last_byte_pos

<commit_msg>Encode HA2() string parameters 'method', 'uri' and 'H(entityBody)'<commit_after># -*- coding: utf-8 -*-

"""
httpbin.helpers
~~~~~~~~~~~~~~~

This module provides helper functions for httpbin.
"""

import json
import base64
from hashlib import md5, sha256
from werkzeug.http import parse_authorization_header

from flask import request, make_response
from six.moves.urllib.parse import urlparse, urlunparse

from .structures import CaseInsensitiveDict

ASCII_ART = """
    -=[ teapot ]=-

       _...._
     .'  _ _ `.
    | ."` ^ `". _,
    \_;`"---"`|//
      |       ;/
      \_     _/
        `\"\"\"`
"""

REDIRECT_LOCATION = '/redirect/1'

ENV_HEADERS = (
    'X-Varnish',
    'X-Request-Start',
    'X-Heroku-Queue-Depth',
    'X-Real-Ip',
    'X-Forwarded-Proto',
    'X-Forwarded-Protocol',
    'X-Forwarded-Ssl',
    'X-Heroku-Queue-Wait-Time',
    'X-Forwarded-For',
    'X-Heroku-Dynos-In-Use',
    'X-Forwarded-For',
    'X-Forwarded-Protocol',
    'X-Forwarded-Port',
    'Runscope-Service'
)

ROBOT_TXT = """User-agent: *
Disallow: /deny
"""

ACCEPTED_MEDIA_TYPES = [
    'image/webp',
    'image/svg+xml',
    'image/jpeg',
    'image/png',
    'image/*'
]

ANGRY_ASCII ="""
          .-''''''-.
        .' _      _ '.
       /   O      O   \\
      :                :
      |                |
      :       __       :
       \  .-"`  `"-.  /
        '.          .'
          '-......-'
     YOU SHOULDN'T BE HERE
"""

def json_safe(string, content_type='application/octet-stream'):
    """Returns JSON-safe version of `string`.

    If `string` is a Unicode string or a valid UTF-8, it is returned unmodified,
    as it can safely be encoded to JSON string.

    If `string` contains raw/binary data, it is Base64-encoded, formatted and
    returned according to "data" URL scheme (RFC2397). Since JSON is not
    suitable for binary data, some additional encoding was necessary; "data"
    URL scheme was chosen for its simplicity.
    """
    try:
        string = string.decode('utf-8')
        json.dumps(string)
        return string
    except (ValueError, TypeError):
        return b''.join([
            b'data:',
            content_type.encode('utf-8'),
            b';base64,',
            base64.b64encode(string)
        ]).decode('utf-8')

def get_files():
    """Returns files dict from request context."""

    files = dict()

    for k, v in request.files.items():
        content_type = request.files[k].content_type or 'application/octet-stream'
        val = json_safe(v.read(), content_type)
        if files.get(k):
            if not isinstance(files[k], list):
                files[k] = [files[k]]
            files[k].append(val)
        else:
            files[k] = val

    return files

def get_headers(hide_env=True):
    """Returns headers dict from request context."""

    headers = dict(request.headers.items())

    if hide_env and ('show_env' not in request.args):
        for key in ENV_HEADERS:
            try:
                del headers[key]
            except KeyError:
                pass

    return CaseInsensitiveDict(headers.items())

def semiflatten(multi):
    """Convert a MutiDict into a regular dict. If there are more than one value
    for a key, the result will have a list of values for the key. Otherwise it
    will have the plain value."""
    if multi:
        result = multi.to_dict(flat=False)
        for k, v in result.items():
            if len(v) == 1:
                result[k] = v[0]
        return result
    else:
        return multi

def get_url(request):
    """
    Since we might be hosted behind a proxy, we need to check the
    X-Forwarded-Proto, X-Forwarded-Protocol, or X-Forwarded-SSL headers
    to find out what protocol was used to access us.
    """
    protocol = request.headers.get('X-Forwarded-Proto') or request.headers.get('X-Forwarded-Protocol')
    if protocol is None and request.headers.get('X-Forwarded-Ssl') == 'on':
        protocol = 'https'
    if protocol is None:
        return request.url
    url = list(urlparse(request.url))
    url[0] = protocol
    return urlunparse(url)

def get_dict(*keys, **extras):
    """Returns request dict of given keys."""

    _keys = ('url', 'args', 'form', 'data', 'origin', 'headers', 'files', 'json')

    assert all(map(_keys.__contains__, keys))
    data = request.data
    form = semiflatten(request.form)

    try:
        _json = json.loads(data.decode('utf-8'))
    except (ValueError, TypeError):
        _json = None

    d = dict(
        url=get_url(request),
        args=semiflatten(request.args),
        form=form,
        data=json_safe(data),
        origin=request.headers.get('X-Forwarded-For', request.remote_addr),
        headers=get_headers(),
        files=get_files(),
        json=_json
    )

    out_d = dict()

    for key in keys:
        out_d[key] = d.get(key)

    out_d.update(extras)

    return out_d

def status_code(code):
    """Returns response object of given status code."""

    redirect = dict(headers=dict(location=REDIRECT_LOCATION))

    code_map = {
        301: redirect,
        302: redirect,
        303: redirect,
        304: dict(data=''),
        305: redirect,
        307: redirect,
        401: dict(headers={'WWW-Authenticate': 'Basic realm="Fake Realm"'}),
        402: dict(
            data='Fuck you, pay me!',
            headers={
                'x-more-info': 'http://vimeo.com/22053820'
            }
        ),
        406: dict(data=json.dumps({
                'message': 'Client did not request a supported media type.',
                'accept': ACCEPTED_MEDIA_TYPES
            }),
            headers={
                'Content-Type': 'application/json'
            }),
        407: dict(headers={'Proxy-Authenticate': 'Basic realm="Fake Realm"'}),
        418: dict(  # I'm a teapot!
            data=ASCII_ART,
            headers={
                'x-more-info': 'http://tools.ietf.org/html/rfc2324'
            }
        ),

    }

    r = make_response()
    r.status_code = code

    if code in code_map:

        m = code_map[code]

        if 'data' in m:
            r.data = m['data']
        if 'headers' in m:
            r.headers = m['headers']

    return r

def check_basic_auth(user, passwd):
    """Checks user authentication using HTTP Basic Auth."""

    auth = request.authorization
    return auth and auth.username == user and auth.password == passwd

# Digest auth helpers
# qop is a quality of protection

def H(data, algorithm):
    if algorithm == 'SHA-256':
        return sha256(data).hexdigest()
    else:
        return md5(data).hexdigest()

def HA1(realm, username, password, algorithm):
    """Create HA1 hash by realm, username, password

    HA1 = md5(A1) = MD5(username:realm:password)
    """
    if not realm:
        realm = u''
    return H(b":".join([username.encode('utf-8'),
                           realm.encode('utf-8'),
                           password.encode('utf-8')]), algorithm)

def HA2(credentails, request, algorithm):
    """Create HA2 md5 hash

    If the qop directive's value is "auth" or is unspecified, then HA2:
        HA2 = md5(A2) = MD5(method:digestURI)
    If the qop directive's value is "auth-int" , then HA2 is
        HA2 = md5(A2) = MD5(method:digestURI:MD5(entityBody))
    """
    if credentails.get("qop") == "auth" or credentails.get('qop') is None:
        return H(b":".join([request['method'].encode('utf-8'), request['uri'].encode('utf-8')]), algorithm)
    elif credentails.get("qop") == "auth-int":
        for k in 'method', 'uri', 'body':
            if k not in request:
                raise ValueError("%s required" % k)
        A2 = b":".join([request['method'].encode('utf-8'),
                        request['uri'].encode('utf-8'),
                        H(request['body'], algorithm).encode('utf-8')])
        return H(A2, algorithm)
    raise ValueError

def response(credentails, password, request):
    """Compile digest auth response

    If the qop directive's value is "auth" or "auth-int" , then compute the response as follows:
       RESPONSE = MD5(HA1:nonce:nonceCount:clienNonce:qop:HA2)
    Else if the qop directive is unspecified, then compute the response as follows:
       RESPONSE = MD5(HA1:nonce:HA2)

    Arguments:
    - `credentails`: credentails dict
    - `password`: request user password
    - `request`: request dict
    """
    response = None
    algorithm = credentails.get('algorithm')
    HA1_value = HA1(
        credentails.get('realm'),
        credentails.get('username'),
        password,
        algorithm
    )
    HA2_value = HA2(credentails, request, algorithm)
    if credentails.get('qop') is None:
        response = H(b":".join([
            HA1_value.encode('utf-8'),
            credentails.get('nonce', '').encode('utf-8'),
            HA2_value.encode('utf-8')
        ]), algorithm)
    elif credentails.get('qop') == 'auth' or credentails.get('qop') == 'auth-int':
        for k in 'nonce', 'nc', 'cnonce', 'qop':
            if k not in credentails:
                raise ValueError("%s required for response H" % k)
        response = H(b":".join([HA1_value.encode('utf-8'),
                               credentails.get('nonce').encode('utf-8'),
                               credentails.get('nc').encode('utf-8'),
                               credentails.get('cnonce').encode('utf-8'),
                               credentails.get('qop').encode('utf-8'),
                               HA2_value.encode('utf-8')]), algorithm)
    else:
        raise ValueError("qop value are wrong")

    return response

def check_digest_auth(user, passwd):
    """Check user authentication using HTTP Digest auth"""

    if request.headers.get('Authorization'):
        credentails = parse_authorization_header(request.headers.get('Authorization'))
        if not credentails:
            return
        response_hash = response(credentails, passwd, dict(uri=request.script_root + request.path,
                                                           body=request.data,
                                                           method=request.method))
        if credentails.get('response') == response_hash:
            return True
    return False

def secure_cookie():
    """Return true if cookie should have secure attribute"""
    return request.environ['wsgi.url_scheme'] == 'https'

def __parse_request_range(range_header_text):
    """ Return a tuple describing the byte range requested in a GET request
    If the range is open ended on the left or right side, then a value of None
    will be set.
    RFC7233: http://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7233.html#header.range
    Examples:
      Range : bytes=1024-
      Range : bytes=10-20
      Range : bytes=-999
    """

    left = None
    right = None

    if not range_header_text:
        return left, right

    range_header_text = range_header_text.strip()
    if not range_header_text.startswith('bytes'):
        return left, right

    components = range_header_text.split("=")
    if len(components) != 2:
        return left, right

    components = components[1].split("-")

    try:
        right = int(components[1])
    except:
        pass

    try:
        left = int(components[0])
    except:
        pass

    return left, right

def get_request_range(request_headers, upper_bound):
    first_byte_pos, last_byte_pos = __parse_request_range(request_headers['range'])

    if first_byte_pos is None and last_byte_pos is None:
        # Request full range
        first_byte_pos = 0
        last_byte_pos = upper_bound - 1
    elif first_byte_pos is None:
        # Request the last X bytes
        first_byte_pos = max(0, upper_bound - last_byte_pos)
        last_byte_pos = upper_bound - 1
    elif last_byte_pos is None:
        # Request the last X bytes
        last_byte_pos = upper_bound - 1

    return first_byte_pos, last_byte_pos

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import argparse
import pymatgen as mp

def make_input(id = 'noid', potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, NSELF_AMOBT = False,
DIEL = False, PHONON = False, DEFORM = False, AMOBT = False, SOC = False, computer = 'partita'):
	mp_api_key = 'fDJKEZpxSyvsXdCt'
	with open('MIKECAR','w') as input:
		input.write('EMAIL = alireza@wustl.edu \n')
		input.write('MP_API_KEY = ' + mp_api_key + ' \n')
		input.write('JOB_NAME = See_Folder \n')
		input.write('COMP_NAME = ' + id + ' \n')
		input.write('%s %s \n' % ('GEOM =', GEOM))
		input.write('%s %s \n' % ('SELF =', SELF))
		input.write('%s %s \n' % ('NSELF =', NSELF))
		input.write('%s %s \n' % ('NSELF_AMOBT =', NSELF_AMOBT))
		input.write('%s %s \n' % ('DIEL =', DIEL))
		input.write('%s %s \n' % ('PHONON =', PHONON))
		api = mp.MPRester(mp_api_key)
		if id not in ['noid']:
			structure = api.get_structure_by_material_id(id)
			structure.to(filename="POSCAR")
			with open('POSCAR','r') as poscar:
				counter = 0
				natoms = 0
				for line in poscar:
					counter += 1
					if counter == 7:
						for i in line.split():
							natoms += int(i)
			os.system('rm POSCAR')
			if natoms < 17:
				input.write('DIM = 2x2x2 \n')
			else:
				input.write('DIM = 1x1x1 \n')
		else:
			input.write('DIM = 1x1x1 \n')
		input.write('%s %s \n' % ('DEFORM =', DEFORM))
		input.write('RANGE = -5,5 \n')
		input.write('POSCAR = default \n')
		input.write('INCAR = default \n')
		input.write('%s %s \n' % ('AMOBT =', AMOBT))
		input.write('%s %s \n' % ('SOC =', SOC))
		input.write('%s %s \n' % ('COMPUTER =', computer))
		input.write('%s %s \n' % ('AMOBT_PATH =', amobt_path))
		input.write('%s %s \n' % ('SCRIPTS_PATH =', scripts_path))
		input.write('%s %s \n' % ('POTCAR_PATH =', potcar_path))

def run_dekode_for(materials_list, potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, NSELF_AMOBT =
False, DIEL = False, PHONON = False, DEFORM = False, AMOBT = False, SOC = False, computer = 'partita'):
	if SOC:
#		if ~os.path.exists('SOC')
		os.system('mkdir SOC')
		os.chdir('SOC')
	for id in materials_list:
## Uncomment #1 if you want to start from the beginning
		os.system('mkdir ' + id)
		os.system('%s %s%s %s' % ('cp', scripts_path, 'dekode.py', id))
		os.system('%s %s%s %s' % ('cp', scripts_path, 'partita.sh', id))
		os.system('cp ~/vasp-ib2.csh ' + id)
		os.chdir(id)
#		make_input(id, amobt_path, scripts_path, SOC) #1
		make_input(id, potcar_path, amobt_path, scripts_path, GEOM, SELF, NSELF, NSELF_AMOBT, DIEL, PHONON, DEFORM, AMOBT,
SOC, computer)
		os.system('rm python_job.*')
		os.system('qsub partita.sh')
		os.chdir('../')

if __name__ == "__main__":

	# The following are the first 9 n-type TCO's
	amobt_path = '/research-projects/partita/faghaniniaa/current_jobs/carrier_scattering/VERSIONS/latest_aMoBT/'
	scripts_path = '~/dekode/'
	potcar_path = '/cluster/caml/vasp-pot/PBE/'

	list1 = ['mp-5966', 'mp-22598', 'mp-19803', 'mp-29213', 'mp-22323', 'mp-22189', 'mp-31132', 'mp-8275', 'mp-13334', 'mp-3443', 'mp-3810', 'mp-16281']
	list2 = ['mp-3188', 'mp-1243', 'mp-504908', 'mp-29297', 'mp-566788', 'mp-10913', 'mp-28931', 'mp-13803', 'mp-5794', 'mp-8285', 'mp-3917', 'mp-19148']
	list3 = ['mp-1705', 'mp-30284', 'mp-886', 'mp-25178', 'mp-27175', 'mp-540728', 'mp-5280', 'mp-19006', 'mp-3056', 'mp-7534']
	list4 = ['mp-8922', 'mp-7762', 'mp-13820', 'mp-20546', 'mp-27843', 'mp-540688', 'mp-27563', 'mp-8086', 'mp-25014', 'mp-19321']
	list5 = ['mp-7831', 'mp-16293', 'mp-3653', 'mp-29910', 'mp-29455', 'mp-25043', 'mp-10486', 'mp-5909', 'mp-22937', 'mp-29606', 'mp-2951']
	list6 = ['mp-4590', 'mp-541368', 'mp-27684', 'mp-22734', 'mp-7461', 'mp-20337', 'mp-23419']
	list7 = ['mp-5380', 'mp-7863', 'mp-8624', 'mp-3744', 'mp-7502', 'mp-1346', 'mp-20694', 'mp-761872']
	sarmandi = ['mp-7233', 'mp-4764']    # X2SeO2, X= La, Pr, Nd, Gd
	zero_gaps = ['mp-13971', 'mp-2045']    # X2SeO2, X=  Pr, Nd, Gd mp-19079 (zero-gap even on MP)
	pTCO_SI_list1 = ['mp-28962', 'mp-13060', 'mp-617']
	pTCO_SI_list2 = ['mp-5709', 'mp-18288', 'mp-28711', 'mp-23487']
	pTCO_SI_list3 = ['mp-25726', 'mp-3370', 'mp-4086', 'mp-2931', 'mp-17559', 'mp-8111']
	TE1 = ['mp-22619', 'mp-486', 'mp-555818', 'mp-2074', 'mp-7955', 'mp-991676', 'mp-9295', 'mp-991652']  # NiP2, NiP2, SiCu2PbS4, Li3Sb, Li3Sb, (Nb-Ta-V)Cu3Te4

	famous = ['mp-2133', 'mp-3163', 'mp-22598', 'mp-22323', 'mp-644741'] # 			ZnO, BaSnO3, 3_In2O3
	famous1= ['mp-856', 'mp-550172', 'mp-755071', 'mp-545544', 'mp-23092', 'mp-552200']  #  3_SnO2, 3_Ba2TaBiO6
	missed = ['mp-5966', 'mp-22598', 'mp-19803', 'mp-29213', 'mp-14243', 'mp-22189', 'mp-8275', 'mp-13334', 'mp-3443', 'mp-3810', 'mp-16281','mp-566788','mp-19321','mp-22189']
#	excluded_first_pass = ['mp-510309', 'mp-22606', 'mp-505501']	# NONE OF THESE COULD BE READ FROM MATERIALSPROJECT
#	566788: n cannot be obtained in aMoBT!
	famous_p = ['mp-10695', 'mp-560588', 'mp-3748', 'mp-3098', 'mp-4979']	# ZnS_2, CuAlO2_2, CuAlS2
	famous_p_2 = ['mp-5970', 'mp-4255', 'mp-19281', 'mp-19357', 'mp-18841']	# BaCu2S2_2, SrxLa1-xCrO3_3

	halides_1 = ['mp-22922', 'mp-22914', 'mp-22913', 'mp-32880','mp-614013', 'mp-616378', 'mp-568570', 'mp-2068']   # p: AgCl, CuCl, CuBr_2  n: CsSnI3_3, RhF3
# mp-21905, mp-3331 and mp-29047 don't exist on MP website
	so_list = ['mp-8298', 'mp-8299', 'mp-7787', 'mp-19845', 'mp-8789', 'mp-3519']
#	materials_list = list1 + list2 + list3 + list4 + list5 + list6 + list7
	materials_list = list2 + list3 + list4 + list5 + list6 + list7 + sarmandi + pTCO_SI_list1 + pTCO_SI_list2 + pTCO_SI_list3 + so_list + zero_gaps + famous + famous1

#	materials_list = ['mp-569522'] # MnP4
#	materials_list = ['mp-16764']  # RbYTe2
#	materials_list = ['mp-16763']  # KYTe2
#	materials_list = ['mp-945184'] # YCyTe2
#	materials_list = ['mp-12953']  # TmAgTe2
#	materials_list = ['mp-4753']  # Al2SiO5
#	materials_list = ['mp-487', 'mp-14587' ,'mp-570553']  # MnP4, ZnP4, FeP4
#	materials_list = ['mp-7623']  # MgAs4
#	materials_list = ['mp-7904']  # CdP4
#	materials_list = ['mp-384']  # MgP4
#	materials_list = ['mp-19717']  # PbTe
#	materials_list = sarmandi
#	materials_list = ['mp-7541']  # SnP3
#	materials_list = pTCO_SI_list3
#	materials_list = ['mp-640889']  # TmCu3Te3
	occasional_TEs = ['mp-569522', 'mp-16764', 'mp-16763', 'mp-945184', 'mp-12953', 'mp-4753', 'mp-487', 'mp-14587' ,'mp-570553', 'mp-7623', 'mp-7904', 'mp-384', 'mp-19717', 'mp-7541', 'mp-640889']
#	materials_list = occasional_TEs + TE1

	SOC = False

# List of manuals:
#		'mp-21905'
#		'mp-13973'

	parser = argparse.ArgumentParser()
	parser.add_argument("-pa","--mp_id", help="The Materials Project ID", required = False)
	args = parser.parse_args()

	run_dekode_for(['mp-691'], potcar_path, amobt_path, scripts_path, GEOM = True, SELF = True, NSELF = True, \
	NSELF_AMOBT = True, DIEL = True, PHONON = True, DEFORM = True, AMOBT = True, SOC = False, computer = 'partita')

#	run_dekode_for(['mp-7831', 'mp-541368', 'mp-22734', 'mp-7863', 'mp-3744', 'mp-7502', 'mp-7233', 'mp-5709'], potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, \
#	NSELF_AMOBT = False, DIEL = False, PHONON = False, DEFORM = False, AMOBT = True, SOC = False, computer = 'partita')

#	run_dekode_for(['mp-16281','mp-566788','mp-19321','mp-22189'], potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, \
#	NSELF_AMOBT = False, DIEL = False, PHONON = False, DEFORM = False, AMOBT = True, SOC = False, computer = 'partita')
<commit_msg>small correction to run_dekode<commit_after>#!/usr/bin/env python

import os
import argparse
import pymatgen as mp

def make_input(potcar_path, amobt_path, scripts_path, id = 'noid',  GEOM = False, SELF = False, NSELF = False, NSELF_AMOBT = False,
DIEL = False, PHONON = False, DEFORM = False, AMOBT = False, SOC = False, computer = 'partita'):
	mp_api_key = 'fDJKEZpxSyvsXdCt'
	with open('MIKECAR','w') as input:
		input.write('EMAIL = alireza@wustl.edu \n')
		input.write('MP_API_KEY = ' + mp_api_key + ' \n')
		input.write('JOB_NAME = See_Folder \n')
		input.write('COMP_NAME = ' + id + ' \n')
		input.write('%s %s \n' % ('GEOM =', GEOM))
		input.write('%s %s \n' % ('SELF =', SELF))
		input.write('%s %s \n' % ('NSELF =', NSELF))
		input.write('%s %s \n' % ('NSELF_AMOBT =', NSELF_AMOBT))
		input.write('%s %s \n' % ('DIEL =', DIEL))
		input.write('%s %s \n' % ('PHONON =', PHONON))
		api = mp.MPRester(mp_api_key)
		if id not in ['noid']:
			structure = api.get_structure_by_material_id(id)
			structure.to(filename="POSCAR")
			with open('POSCAR','r') as poscar:
				counter = 0
				natoms = 0
				for line in poscar:
					counter += 1
					if counter == 7:
						for i in line.split():
							natoms += int(i)
			os.system('rm POSCAR')
			if natoms < 17:
				input.write('DIM = 2x2x2 \n')
			else:
				input.write('DIM = 1x1x1 \n')
		else:
			input.write('DIM = 1x1x1 \n')
		input.write('%s %s \n' % ('DEFORM =', DEFORM))
		input.write('RANGE = -5,5 \n')
		input.write('POSCAR = default \n')
		input.write('INCAR = default \n')
		input.write('%s %s \n' % ('AMOBT =', AMOBT))
		input.write('%s %s \n' % ('SOC =', SOC))
		input.write('%s %s \n' % ('COMPUTER =', computer))
		input.write('%s %s \n' % ('AMOBT_PATH =', amobt_path))
		input.write('%s %s \n' % ('SCRIPTS_PATH =', scripts_path))
		input.write('%s %s \n' % ('POTCAR_PATH =', potcar_path))

def run_dekode_for(materials_list, potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, NSELF_AMOBT =
False, DIEL = False, PHONON = False, DEFORM = False, AMOBT = False, SOC = False, computer = 'partita'):
	if SOC:
#		if ~os.path.exists('SOC')
		os.system('mkdir SOC')
		os.chdir('SOC')
	for id in materials_list:
## Uncomment #1 if you want to start from the beginning
		os.system('mkdir ' + id)
		os.system('%s %s%s %s' % ('cp', scripts_path, 'dekode.py', id))
		os.system('%s %s%s %s' % ('cp', scripts_path, 'partita.sh', id))
		os.system('cp ~/vasp-ib2.csh ' + id)
		os.chdir(id)
#		make_input(id, amobt_path, scripts_path, SOC) #1
		make_input(potcar_path, amobt_path, scripts_path, id, GEOM, SELF, NSELF, NSELF_AMOBT, DIEL, PHONON, DEFORM, AMOBT,
SOC, computer)
		os.system('rm python_job.*')
		os.system('qsub partita.sh')
		os.chdir('../')

if __name__ == "__main__":

	# The following are the first 9 n-type TCO's
	amobt_path = '/research-projects/partita/faghaniniaa/current_jobs/carrier_scattering/VERSIONS/latest_aMoBT/'
	scripts_path = '~/dekode/'
	potcar_path = '/cluster/caml/vasp-pot/PBE/'

	list1 = ['mp-5966', 'mp-22598', 'mp-19803', 'mp-29213', 'mp-22323', 'mp-22189', 'mp-31132', 'mp-8275', 'mp-13334', 'mp-3443', 'mp-3810', 'mp-16281']
	list2 = ['mp-3188', 'mp-1243', 'mp-504908', 'mp-29297', 'mp-566788', 'mp-10913', 'mp-28931', 'mp-13803', 'mp-5794', 'mp-8285', 'mp-3917', 'mp-19148']
	list3 = ['mp-1705', 'mp-30284', 'mp-886', 'mp-25178', 'mp-27175', 'mp-540728', 'mp-5280', 'mp-19006', 'mp-3056', 'mp-7534']
	list4 = ['mp-8922', 'mp-7762', 'mp-13820', 'mp-20546', 'mp-27843', 'mp-540688', 'mp-27563', 'mp-8086', 'mp-25014', 'mp-19321']
	list5 = ['mp-7831', 'mp-16293', 'mp-3653', 'mp-29910', 'mp-29455', 'mp-25043', 'mp-10486', 'mp-5909', 'mp-22937', 'mp-29606', 'mp-2951']
	list6 = ['mp-4590', 'mp-541368', 'mp-27684', 'mp-22734', 'mp-7461', 'mp-20337', 'mp-23419']
	list7 = ['mp-5380', 'mp-7863', 'mp-8624', 'mp-3744', 'mp-7502', 'mp-1346', 'mp-20694', 'mp-761872']
	sarmandi = ['mp-7233', 'mp-4764']    # X2SeO2, X= La, Pr, Nd, Gd
	zero_gaps = ['mp-13971', 'mp-2045']    # X2SeO2, X=  Pr, Nd, Gd mp-19079 (zero-gap even on MP)
	pTCO_SI_list1 = ['mp-28962', 'mp-13060', 'mp-617']
	pTCO_SI_list2 = ['mp-5709', 'mp-18288', 'mp-28711', 'mp-23487']
	pTCO_SI_list3 = ['mp-25726', 'mp-3370', 'mp-4086', 'mp-2931', 'mp-17559', 'mp-8111']
	TE1 = ['mp-22619', 'mp-486', 'mp-555818', 'mp-2074', 'mp-7955', 'mp-991676', 'mp-9295', 'mp-991652']  # NiP2, NiP2, SiCu2PbS4, Li3Sb, Li3Sb, (Nb-Ta-V)Cu3Te4

	famous = ['mp-2133', 'mp-3163', 'mp-22598', 'mp-22323', 'mp-644741'] # 			ZnO, BaSnO3, 3_In2O3
	famous1= ['mp-856', 'mp-550172', 'mp-755071', 'mp-545544', 'mp-23092', 'mp-552200']  #  3_SnO2, 3_Ba2TaBiO6
	missed = ['mp-5966', 'mp-22598', 'mp-19803', 'mp-29213', 'mp-14243', 'mp-22189', 'mp-8275', 'mp-13334', 'mp-3443', 'mp-3810', 'mp-16281','mp-566788','mp-19321','mp-22189']
#	excluded_first_pass = ['mp-510309', 'mp-22606', 'mp-505501']	# NONE OF THESE COULD BE READ FROM MATERIALSPROJECT
#	566788: n cannot be obtained in aMoBT!
	famous_p = ['mp-10695', 'mp-560588', 'mp-3748', 'mp-3098', 'mp-4979']	# ZnS_2, CuAlO2_2, CuAlS2
	famous_p_2 = ['mp-5970', 'mp-4255', 'mp-19281', 'mp-19357', 'mp-18841']	# BaCu2S2_2, SrxLa1-xCrO3_3

	halides_1 = ['mp-22922', 'mp-22914', 'mp-22913', 'mp-32880','mp-614013', 'mp-616378', 'mp-568570', 'mp-2068']   # p: AgCl, CuCl, CuBr_2  n: CsSnI3_3, RhF3
# mp-21905, mp-3331 and mp-29047 don't exist on MP website
	so_list = ['mp-8298', 'mp-8299', 'mp-7787', 'mp-19845', 'mp-8789', 'mp-3519']
#	materials_list = list1 + list2 + list3 + list4 + list5 + list6 + list7
	materials_list = list2 + list3 + list4 + list5 + list6 + list7 + sarmandi + pTCO_SI_list1 + pTCO_SI_list2 + pTCO_SI_list3 + so_list + zero_gaps + famous + famous1

#	materials_list = ['mp-569522'] # MnP4
#	materials_list = ['mp-16764']  # RbYTe2
#	materials_list = ['mp-16763']  # KYTe2
#	materials_list = ['mp-945184'] # YCyTe2
#	materials_list = ['mp-12953']  # TmAgTe2
#	materials_list = ['mp-4753']  # Al2SiO5
#	materials_list = ['mp-487', 'mp-14587' ,'mp-570553']  # MnP4, ZnP4, FeP4
#	materials_list = ['mp-7623']  # MgAs4
#	materials_list = ['mp-7904']  # CdP4
#	materials_list = ['mp-384']  # MgP4
#	materials_list = ['mp-19717']  # PbTe
#	materials_list = sarmandi
#	materials_list = ['mp-7541']  # SnP3
#	materials_list = pTCO_SI_list3
#	materials_list = ['mp-640889']  # TmCu3Te3
	occasional_TEs = ['mp-569522', 'mp-16764', 'mp-16763', 'mp-945184', 'mp-12953', 'mp-4753', 'mp-487', 'mp-14587' ,'mp-570553', 'mp-7623', 'mp-7904', 'mp-384', 'mp-19717', 'mp-7541', 'mp-640889']
#	materials_list = occasional_TEs + TE1

	SOC = False

# List of manuals:
#		'mp-21905'
#		'mp-13973'

	parser = argparse.ArgumentParser()
	parser.add_argument("-pa","--mp_id", help="The Materials Project ID", required = False)
	args = parser.parse_args()

	run_dekode_for(['mp-691'], potcar_path, amobt_path, scripts_path, GEOM = True, SELF = True, NSELF = True, \
	NSELF_AMOBT = True, DIEL = True, PHONON = True, DEFORM = True, AMOBT = True, SOC = False, computer = 'partita')

#	run_dekode_for(['mp-7831', 'mp-541368', 'mp-22734', 'mp-7863', 'mp-3744', 'mp-7502', 'mp-7233', 'mp-5709'], potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, \
#	NSELF_AMOBT = False, DIEL = False, PHONON = False, DEFORM = False, AMOBT = True, SOC = False, computer = 'partita')

#	run_dekode_for(['mp-16281','mp-566788','mp-19321','mp-22189'], potcar_path, amobt_path, scripts_path, GEOM = False, SELF = False, NSELF = False, \
#	NSELF_AMOBT = False, DIEL = False, PHONON = False, DEFORM = False, AMOBT = True, SOC = False, computer = 'partita')
<|endoftext|>


<commit_before>#!/usr/bin/python
"""Make a pdf from the specified book."""
import os, sys
import cgi
import re
import tempfile
from urllib2 import urlopen
from getopt import gnu_getopt
from subprocess import Popen, check_call

import lxml.etree, lxml.html

SERVER_NAME = os.environ.get('SERVER_NAME', 'en.flossmanuals.net')

TOC_URL = "http://%s/pub/%%s/_index/TOC.txt" % SERVER_NAME
BOOK_URL = "http://%s/bin/view/%%s/_all?skin=text" % SERVER_NAME
ADD_TEMPLATE = True

DEFAULT_CSS = 'file://' + os.path.abspath('default.css')

# ARG_VALIDATORS is a mapping between the expected cgi arguments and
# functions to validate their values. (None means no validation).
ARG_VALIDATORS = {
    "webName": lambda x: '/' not in x and '..' not in x,
    "css": None,
    "title": None,
    "header": None,
    "isbn": None,
    "license": None,
}

__doc__ += '\nValid arguments are: %s.\n' % ', '.join(ARG_VALIDATORS.keys())

_files_to_rm = []

def save_tempfile(data, **kwargs):
    """Save the data in a tempfile and put the filename on the
    clean-up list.  A sensible keyword argument might be
    suffix='.css'."""
    fh, fn = tempfile.mkstemp(**kwargs)
    if isinstance(data, unicode):
        data = data.encode('utf8', 'ignore')
    os.write(fh, data)
    os.close(fh)
    _files_to_rm.append(fn)
    return os.path.abspath(fn)

def log(*messages):
    for m in messages:
        print >> sys.stderr, m

def parse_args():
    """Read and validate CGI or commandline arguments, putting the
    good ones into the returned dictionary.  Command line arguments
    should be in the form --title='A Book'.
    """
    query = cgi.FieldStorage()
    options, args = gnu_getopt(sys.argv[1:], '', [x + '=' for x in ARG_VALIDATORS])
    print options
    options = dict(options)
    print options
    data = {}
    for key, validator in ARG_VALIDATORS.items():
        value = query.getfirst(key, options.get('--' + key, None))
        print key, value
        if value is not None:
            if validator is not None and not validator(value):
                log("argument '%s' is not valid ('%s')" % (key, value))
                continue
            data[key] = value

    print data
    return data

def toc_reader(name):
    """Iterate over the TOC.txt file.  For each chapter yield the
    tuple (<status>, <filename>, <title>).

    <status> is a number, with the following meaning:

          0 - section heading with no chapter
          1 - chapter heading
          2 - book title

    <filename> is twiki name of the chapter.

    <title> is a human readable title for the chapter.  It is likely to
    differ from the title given in the chapter's
 heading.
    """
    f = urlopen(TOC_URL % name)
    while True:
        try:
            yield (f.next().strip(), f.next().strip(), f.next().strip())
        except StopIteration:
            break
    f.close()

def get_book(name, tidy=True):
    """Fetch and parse the raw html of the book.  If tidy is true
    (default) links in the document will be made absolute."""
    f = urlopen(BOOK_URL % name)
    html = f.read()
    if ADD_TEMPLATE:
        html = ('\n<title>%s</title>\n'
                '<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />\n'
                '\n\n'
                '%s\n') % (name, html)
    f.close()
    f = open('/tmp/%s-RAW.html' % name, 'w')
    f.write(html)
    f.close()
    tree = lxml.html.document_fromstring(html)
    if tidy:
        tree.make_links_absolute(BOOK_URL % name)

    return tree

class PageSettings:
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    def wkcommand(self, html, pdf):
        m = [str(x) for x in self.wkmargins]
        cmd = ['wkhtmltopdf', '-s', self.wksize,
               '-T', m[0], '-R', m[1], '-B', m[2], '-L', m[3],
               html, pdf
               ]
        log(' '.join(cmd))
        return cmd

    def shiftcommand(self, pdf, dir='LTR', numbers='latin', number_start=1):
        #numbers should be 'latin', 'roman', or 'arabic'
        number_start = str(number_start).replace('-', '_')
        cmd = ['pdfedit', '-s', 'shift_margins', 'shift_margins',
               str(self.shift), pdf, self.name, dir, numbers, number_start]
        log(' '.join(cmd))
        return cmd

    def output_name(self, input_name):
        """replicate the name mangling performed by shift_margins.qs
        pdfedit script."""
        #XXX should just pass in name to pdfedit
        if hasattr(self, 'dir'):
            return '%s-%s-%s.pdf' % (input_name[:-4], self.name, self.dir)
        return '%s-%s.pdf' % (input_name[:-4], self.name)

SIZE_MODES = {
    # name      --> should be the same as the key
    # wksize    --> the size name for wkhtml2pdf
    # wkmargins --> margins for wkhtml2pdf (css style, clockwise from top)
    # shift     --> how many points to shift each page left or right.

    'COMICBOOK' : PageSettings(name='COMICBOOK',
                               wksize='B5',
                               wkmargins=[20, 30, 20, 30],
                               shift=20,
                               )
}

def make_pdf(html_file, pdf_file, size='COMICBOOK', numbers='latin', dir='LTR', number_start=1):
    """Make a pdf of the named html file, using webkit.  Returns a
    filename for the finished PDF."""
    settings = SIZE_MODES[size]
    check_call(settings.wkcommand(html_file, pdf_file))
    check_call(settings.shiftcommand(pdf_file, numbers=numbers,
                                     dir=dir, number_start=number_start))
    return settings.output_name(pdf_file)

def make_pdf_cached(bookid, size='COMICBOOK'):
    #Assume the html is already there
    """Make a pdf of the HTML, using webkit"""
    settings = SIZE_MODES[size]
    html_file = '/tmp/%s.html' % bookid
    pdf_raw = '/tmp/%s.pdf' % bookid
    pdf_shifted = settings.output_name(pdf_raw)
    check_call(settings.wkcommand(html_file, pdf_raw))
    check_call(settings.shiftcommand(pdf_raw))

def find_page(element, start=1):
    """Search through the main PDF and retunr the page on which the
    element occurs.  If start is given, the search begins on that
    page."""
    #XXX do it really.
    import random
    return start + random.randrange(1,4)

def make_contents(htmltree, toc):
    header_tmpl = '
%s

\n'
    row_tmpl = ('
%s	%s	'
                '%s
\n')
    section_tmpl = ('
	%s
\n')
    footer = '\n
TOC ends here'

    contents = []
    headers = []

    chapter = 1
    page_num = 1
    subsections = [] # for the subsection heading pages.

    headings = iter(htmltree.cssselect('h1'))

    for status, chapter_url, text in toc:
        # status is
        #  0 - section heading with no chapter
        #  1 - chapter heading
        #  2 - book title
        #
        # chapter_url is twiki name of the chapter
        # text is a human readable name of the chapter.
        if status == '2':
            headers.append(header_tmpl % text)

        elif status == '1':
            h1 = headings.next()
            title = h1.text_content()
            page_num = find_page(h1, page_num)
            contents.append(row_tmpl % (chapter, title, page_num))
            #put a bold number at the beginning of the h1
            ## #XXX this has to go below! in add_section_titles!
            ## initial = h1.makeelement("strong", Class="initial")
            ## h1.insert(0, initial)
            ## initial.tail = h1.text
            ## initial.text = "%s." % chapter
            ## h1.text = ''

            chapter += 1

        elif status == '0':
            contents.append(section_tmpl % text)
            #XXX interstitial pages need to be made with the main PDF

        else:
            log("mystery TOC line: %s %s %s" % (status, chapter_url, text))

    doc = header + '\n'.join(contents) + footer
    return doc

def make_body_pdf(htmltree, bookid, size='COMICBOOK', numbers='latin'):
    """Make a pdf of the HTML, using webkit"""
    html_file = '/tmp/%s.html' % bookid
    pdf_file = '/tmp/%s.pdf' % bookid
    html_text = lxml.etree.tostring(htmltree, method="html")
    f = open(html_file, 'w')
    f.write(html_text)
    f.close()
    return make_pdf(html_file, pdf_file, size='COMICBOOK', numbers='latin')

def make_preamble_pdf(htmltree, toc, title, css):

    contents = make_contents(htmltree, toc)

    html = ('\n'
            '<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />\n'
            '<link rel="stylesheet" href="%s" />\n'
            '\n\n'
            '
%s
'
            '
%s
\n'
            '
%s
\n') % (css, title, copyright(), contents)

    html_file = save_tempfile(html, suffix='.html')
    pdf_file = html_file[:-5] + '.pdf'
    return make_pdf(html_file, pdf_file, size='COMICBOOK', numbers='roman', number_start=-2)

def _add_initial_number(e, n):
    """Put a styled chapter number n at the beginning of element e."""
    initial = e.makeelement("strong", Class="initial")
    e.insert(0, initial)
    initial.tail = ' '
    if e.text is not None:
        initial.tail += e.text
    e.text = ''
    initial.text = "%s." % n

def add_section_titles(htmltree, toc):
    headings = iter(htmltree.cssselect('h1'))
    chapter = 1
    section = None

    for status, chapter_url, text in toc:
        if status == '1' and section is not None:
            #  chapter heading
            h1 = headings.next()
            title = h1.text_content()
            item = h1.makeelement('div', Class='chapter')
            item.text = title
            _add_initial_number(item, chapter)

            section.append(item)

            if not section.getparent(): #XXX how to tell that the section has not been placed?
                print "placing section"
                h1.addprevious(section)
            else:
                print "NOT placing section"

            #put a bold number at the beginning of the h1
            _add_initial_number(h1, chapter)
            chapter += 1

        elif status == '0':
            section = htmltree.makeelement('div', Class="subsection")
            # it would be natural to use h1 here, but that would muck
            # up the h1 iterator. (original Pisa Objavi uses <h0>).
            heading = lxml.etree.SubElement(section, 'div', Class="subsection-heading")
            heading.text = text

def add_css(htmltree, css=None):
    """If css looks like a url, use it as a stylesheet link.
    Otherwise it is the CSS itself, which is saved to a temporary file
    and linked to."""
    if css is None:
        url = DEFAULT_CSS
    elif not re.match(r'^http://\S+$', css):
        fn = save_tempfile(css, suffix='.css')
        url = 'file://' + fn
    else:
        url = css

    #find the head -- it's probably first child but lets not assume.
    for child in htmltree:
        if child.tag == head:
            head = child
            break
    else:
        head = htmltree.makeelement('head')
        htmltree.insert(0, head)

    link = lxml.etree.SubElement(head, 'link', rel='stylesheet', type='text/css', href=url)

def get_title(htmltree, args):
    if 'title' in args:
        return args['title']
    headings = htmltree.cssselect('title')
    if headings:
        return headings[0].text_content()
    #oh well
    return 'A Manual About ' + args.get('webName', 'Something')

def concat_pdfs(name, *args):
    """Join all the named pdfs together into one and save it as <name>"""
    cmd = ['pdftk']
    cmd.extend(args)
    cmd += ['cat', 'output', name]
    check_call(cmd)

if __name__ == '__main__':
    try:
        args = parse_args()
        web_name = args['webName']

        #make_pdf_cached(web_name)
        #sys.exit()

        title = get_title(html_tree, args)

        htmltree = get_book(web_name)
        toc = list(toc_reader(web_name))

        add_section_titles(htmltree, toc)
        pdfname = make_body_pdf(htmltree, web_name)
        preamble = make_preamble_pdf(htmltree, toc, title, css_url)
        final = concat_pdfs('/tmp/%s-final.pdf' % web_name, preamble, pdfname)
        add_css(htmltree, args.get('css'))

    finally:
        # clean up temp files
        for fn in _files_to_rm:
            os.remove(fn)

<commit_msg>Improve contents tables<commit_after>#!/usr/bin/python
"""Make a pdf from the specified book."""
import os, sys
import cgi
import re
import tempfile
from urllib2 import urlopen
from getopt import gnu_getopt
from subprocess import Popen, check_call

import lxml.etree, lxml.html

SERVER_NAME = os.environ.get('SERVER_NAME', 'en.flossmanuals.net')

TOC_URL = "http://%s/pub/%%s/_index/TOC.txt" % SERVER_NAME
BOOK_URL = "http://%s/bin/view/%%s/_all?skin=text" % SERVER_NAME
ADD_TEMPLATE = True

DEFAULT_CSS = 'file://' + os.path.abspath('default.css')

# ARG_VALIDATORS is a mapping between the expected cgi arguments and
# functions to validate their values. (None means no validation).
ARG_VALIDATORS = {
    "webName": lambda x: '/' not in x and '..' not in x,
    "css": None,
    "title": None,
    "header": None,
    "isbn": None,
    "license": None,
}

__doc__ += '\nValid arguments are: %s.\n' % ', '.join(ARG_VALIDATORS.keys())

_files_to_rm = []

def save_tempfile(data, **kwargs):
    """Save the data in a tempfile and put the filename on the
    clean-up list.  A sensible keyword argument might be
    suffix='.css'."""
    fh, fn = tempfile.mkstemp(**kwargs)
    if isinstance(data, unicode):
        data = data.encode('utf8', 'ignore')
    os.write(fh, data)
    os.close(fh)
    _files_to_rm.append(fn)
    return os.path.abspath(fn)

def log(*messages):
    for m in messages:
        print >> sys.stderr, m

def parse_args():
    """Read and validate CGI or commandline arguments, putting the
    good ones into the returned dictionary.  Command line arguments
    should be in the form --title='A Book'.
    """
    query = cgi.FieldStorage()
    options, args = gnu_getopt(sys.argv[1:], '', [x + '=' for x in ARG_VALIDATORS])
    print options
    options = dict(options)
    print options
    data = {}
    for key, validator in ARG_VALIDATORS.items():
        value = query.getfirst(key, options.get('--' + key, None))
        print key, value
        if value is not None:
            if validator is not None and not validator(value):
                log("argument '%s' is not valid ('%s')" % (key, value))
                continue
            data[key] = value

    print data
    return data

def toc_reader(name):
    """Iterate over the TOC.txt file.  For each chapter yield the
    tuple (<status>, <filename>, <title>).

    <status> is a number, with the following meaning:

          0 - section heading with no chapter
          1 - chapter heading
          2 - book title

    <filename> is twiki name of the chapter.

    <title> is a human readable title for the chapter.  It is likely to
    differ from the title given in the chapter's
 heading.
    """
    f = urlopen(TOC_URL % name)
    while True:
        try:
            yield (f.next().strip(), f.next().strip(), f.next().strip())
        except StopIteration:
            break
    f.close()

def get_book(name, tidy=True):
    """Fetch and parse the raw html of the book.  If tidy is true
    (default) links in the document will be made absolute."""
    f = urlopen(BOOK_URL % name)
    html = f.read()
    if ADD_TEMPLATE:
        html = ('\n<title>%s</title>\n'
                '<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />\n'
                '\n\n'
                '%s\n') % (name, html)
    f.close()
    f = open('/tmp/%s-RAW.html' % name, 'w')
    f.write(html)
    f.close()
    tree = lxml.html.document_fromstring(html)
    if tidy:
        tree.make_links_absolute(BOOK_URL % name)

    return tree

class PageSettings:
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    def wkcommand(self, html, pdf):
        m = [str(x) for x in self.wkmargins]
        cmd = ['wkhtmltopdf', '-s', self.wksize,
               '-T', m[0], '-R', m[1], '-B', m[2], '-L', m[3],
               html, pdf
               ]
        log(' '.join(cmd))
        return cmd

    def shiftcommand(self, pdf, dir='LTR', numbers='latin', number_start=1):
        #numbers should be 'latin', 'roman', or 'arabic'
        number_start = str(number_start).replace('-', '_')
        cmd = ['pdfedit', '-s', 'shift_margins', 'shift_margins',
               str(self.shift), pdf, self.name, dir, numbers, number_start]
        log(' '.join(cmd))
        return cmd

    def output_name(self, input_name):
        """replicate the name mangling performed by shift_margins.qs
        pdfedit script."""
        #XXX should just pass in name to pdfedit
        if hasattr(self, 'dir'):
            return '%s-%s-%s.pdf' % (input_name[:-4], self.name, self.dir)
        return '%s-%s.pdf' % (input_name[:-4], self.name)

SIZE_MODES = {
    # name      --> should be the same as the key
    # wksize    --> the size name for wkhtml2pdf
    # wkmargins --> margins for wkhtml2pdf (css style, clockwise from top)
    # shift     --> how many points to shift each page left or right.

    'COMICBOOK' : PageSettings(name='COMICBOOK',
                               wksize='B5',
                               wkmargins=[20, 30, 20, 30],
                               shift=20,
                               )
}

def make_pdf(html_file, pdf_file, size='COMICBOOK', numbers='latin', dir='LTR', number_start=1):
    """Make a pdf of the named html file, using webkit.  Returns a
    filename for the finished PDF."""
    settings = SIZE_MODES[size]
    check_call(settings.wkcommand(html_file, pdf_file))
    check_call(settings.shiftcommand(pdf_file, numbers=numbers,
                                     dir=dir, number_start=number_start))
    return settings.output_name(pdf_file)

def make_pdf_cached(bookid, size='COMICBOOK'):
    #Assume the html is already there
    """Make a pdf of the HTML, using webkit"""
    settings = SIZE_MODES[size]
    html_file = '/tmp/%s.html' % bookid
    pdf_raw = '/tmp/%s.pdf' % bookid
    pdf_shifted = settings.output_name(pdf_raw)
    check_call(settings.wkcommand(html_file, pdf_raw))
    check_call(settings.shiftcommand(pdf_raw))

def find_page(element, start=1):
    """Search through the main PDF and retunr the page on which the
    element occurs.  If start is given, the search begins on that
    page."""
    #XXX do it really.
    import random
    return start + random.randrange(1,4)

def make_contents(htmltree, toc):
    header = '
Table of Contents

\n'
    row_tmpl = ('
%s	%s	'
                '%s
\n')
    section_tmpl = ('
%s
\n')
    footer = '\n
'

    contents = []

    chapter = 1
    page_num = 1
    subsections = [] # for the subsection heading pages.

    headings = iter(htmltree.cssselect('h1'))

    for status, chapter_url, text in toc:
        # status is
        #  0 - section heading with no chapter
        #  1 - chapter heading
        #  2 - book title
        #
        # chapter_url is twiki name of the chapter
        # text is a human readable name of the chapter.
        if status == '1':
            h1 = headings.next()
            title = h1.text_content().split('.', 1)[1]
            page_num = find_page(h1, page_num)
            contents.append(row_tmpl % (chapter, title, page_num))
            #put a bold number at the beginning of the h1
            ## #XXX this has to go below! in add_section_titles!
            ## initial = h1.makeelement("strong", Class="initial")
            ## h1.insert(0, initial)
            ## initial.tail = h1.text
            ## initial.text = "%s." % chapter
            ## h1.text = ''

            chapter += 1

        elif status == '0':
            contents.append(section_tmpl % text)
            #XXX interstitial pages need to be made with the main PDF

        else:
            log("mystery TOC line: %s %s %s" % (status, chapter_url, text))

    doc = header + '\n'.join(contents) + footer
    return doc

def make_body_pdf(htmltree, bookid, size='COMICBOOK', numbers='latin'):
    """Make a pdf of the HTML, using webkit"""
    html_file = '/tmp/%s.html' % bookid
    pdf_file = '/tmp/%s.pdf' % bookid
    html_text = lxml.etree.tostring(htmltree, method="html")
    f = open(html_file, 'w')
    f.write(html_text)
    f.close()
    return make_pdf(html_file, pdf_file, size='COMICBOOK', numbers='latin')

def make_preamble_pdf(htmltree, toc, title, css):

    contents = make_contents(htmltree, toc)

    html = ('\n'
            '<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />\n'
            '<link rel="stylesheet" href="%s" />\n'
            '\n\n'
            '
%s
'
            '
%s
\n'
            '
%s
\n') % (css, title, copyright(), contents)

    html_file = save_tempfile(html, suffix='.html')
    pdf_file = html_file[:-5] + '.pdf'
    return make_pdf(html_file, pdf_file, size='COMICBOOK', numbers='roman', number_start=-2)

def _add_initial_number(e, n):
    """Put a styled chapter number n at the beginning of element e."""
    initial = e.makeelement("strong", Class="initial")
    e.insert(0, initial)
    initial.tail = ' '
    if e.text is not None:
        initial.tail += e.text
    e.text = ''
    initial.text = "%s." % n

def add_section_titles(htmltree, toc):
    headings = iter(htmltree.cssselect('h1'))
    chapter = 1
    section = None

    for status, chapter_url, text in toc:
        if status == '1' and section is not None:
            #  chapter heading
            h1 = headings.next()
            title = h1.text_content()
            item = h1.makeelement('div', Class='chapter')
            item.text = title
            _add_initial_number(item, chapter)

            section.append(item)

            if not section.getparent(): #XXX how to tell that the section has not been placed?
                print "placing section"
                h1.addprevious(section)
            else:
                print "NOT placing section"

            #put a bold number at the beginning of the h1
            _add_initial_number(h1, chapter)
            chapter += 1

        elif status == '0':
            section = htmltree.makeelement('div', Class="subsection")
            # it would be natural to use h1 here, but that would muck
            # up the h1 iterator. (original Pisa Objavi uses <h0>).
            heading = lxml.etree.SubElement(section, 'div', Class="subsection-heading")
            heading.text = text

def add_css(htmltree, css=None):
    """If css looks like a url, use it as a stylesheet link.
    Otherwise it is the CSS itself, which is saved to a temporary file
    and linked to."""
    if css is None:
        url = DEFAULT_CSS
    elif not re.match(r'^http://\S+$', css):
        fn = save_tempfile(css, suffix='.css')
        url = 'file://' + fn
    else:
        url = css

    #find the head -- it's probably first child but lets not assume.
    for child in htmltree:
        if child.tag == head:
            head = child
            break
    else:
        head = htmltree.makeelement('head')
        htmltree.insert(0, head)

    link = lxml.etree.SubElement(head, 'link', rel='stylesheet', type='text/css', href=url)

def get_title(htmltree, args):
    if 'title' in args:
        return args['title']
    headings = htmltree.cssselect('title')
    if headings:
        return headings[0].text_content()
    #oh well
    return 'A Manual About ' + args.get('webName', 'Something')

def concat_pdfs(name, *args):
    """Join all the named pdfs together into one and save it as <name>"""
    cmd = ['pdftk']
    cmd.extend(args)
    cmd += ['cat', 'output', name]
    check_call(cmd)

if __name__ == '__main__':
    try:
        args = parse_args()
        web_name = args['webName']

        #make_pdf_cached(web_name)
        #sys.exit()

        title = get_title(html_tree, args)

        htmltree = get_book(web_name)
        toc = list(toc_reader(web_name))

        add_section_titles(htmltree, toc)
        pdfname = make_body_pdf(htmltree, web_name)
        preamble = make_preamble_pdf(htmltree, toc, title, css_url)
        final = concat_pdfs('/tmp/%s-final.pdf' % web_name, preamble, pdfname)
        add_css(htmltree, args.get('css'))

    finally:
        # clean up temp files
        for fn in _files_to_rm:
            os.remove(fn)

<|endoftext|>