<commit_before># encoding: utf-8
import os
import djcelery
from kombu.common import Broadcast
from kombu import Exchange, Queue

djcelery.setup_loader()

_ = gettext = lambda s: s

DEBUG = True
TEMPLATE_DEBUG = DEBUG
TASTYPIE_FULL_DEBUG = True
LOG_DEBUG = False

PRODUCTION_DOMAIN = 'readthedocs.org'
USE_SUBDOMAIN = False

ADMINS = (
    ('Eric Holscher', 'eric@readthedocs.org'),
    ('Anthony Johnson', 'anthony@readthedocs.org'),
)

MANAGERS = ADMINS

SITE_ROOT = '/'.join(os.path.dirname(os.path.realpath(__file__)).split('/')[0:-2])
DOCROOT = os.path.join(SITE_ROOT, 'user_builds')
UPLOAD_ROOT = os.path.join(SITE_ROOT, 'user_uploads')
CNAME_ROOT = os.path.join(SITE_ROOT, 'cnames')
LOGS_ROOT = os.path.join(SITE_ROOT, 'logs')

# A new base for production files
PRODUCTION_ROOT = os.path.join(SITE_ROOT, 'prod_artifacts')
PRODUCTION_MEDIA_ARTIFACTS = os.path.join(PRODUCTION_ROOT, 'media')

MEDIA_ROOT = '%s/media/' % (SITE_ROOT)
MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/media/admin/'

GROK_API_HOST = 'https://api.grokthedocs.com'

# For 1.4
STATIC_ROOT = os.path.join(SITE_ROOT, 'media/static/')
STATIC_URL = '/static/'
STATICFILES_DIRS = [os.path.join(SITE_ROOT, 'readthedocs', 'static')]
#STATICFILES_FINDERS = ()

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'PREFIX': 'docs',
    }
}

CACHE_MIDDLEWARE_SECONDS = 60

LOGIN_REDIRECT_URL = '/dashboard/'
FORCE_WWW = False
#APPEND_SLASH = False

# Docker
DOCKER_ENABLE = False
DOCKER_IMAGE = 'rtfd-build'

TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
LANGUAGES = (
    ('en', gettext('English')),
    ('es', gettext('Spanish')),
    ('nb', gettext('Norwegian Bokmål')),
    ('fr', gettext('French')),
    ('ru', gettext('Russian')),
    ('de', gettext('German')),
    ('gl', gettext('Galician')),
    ('vi', gettext('Vietnamese')),
    ('zh-cn', gettext('Chinese')),
    ('zh-tw', gettext('Taiwanese')),
    ('ja', gettext('Japanese')),
    ('uk', gettext('Ukrainian')),
    ('it', gettext('Italian')),
)
LOCALE_PATHS = [
    os.path.join(SITE_ROOT, 'readthedocs', 'locale'),
]

USE_I18N = True
USE_L10N = True
SITE_ID = 1
SECRET_KEY = 'replace-this-please'

ACCOUNT_ACTIVATION_DAYS = 7

ATOMIC_REQUESTS = True

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'djangosecure.middleware.SecurityMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'pagination.middleware.PaginationMiddleware',
    # Hack
    # 'core.underscore_middleware.UnderscoreMiddleware',
    'core.middleware.SubdomainMiddleware',
    'core.middleware.SingleVersionMiddleware',
    'corsheaders.middleware.CorsMiddleware',
    #'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
)

AUTHENTICATION_BACKENDS = (
    # Needed to login by username in Django admin, regardless of `allauth`
    "django.contrib.auth.backends.ModelBackend",
    # `allauth` specific authentication methods, such as login by e-mail
    "allauth.account.auth_backends.AuthenticationBackend",
)

# All auth

ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = "mandatory"
ACCOUNT_AUTHENTICATION_METHOD = "username_email"
SOCIALACCOUNT_EMAIL_VERIFICATION = "none"

CORS_ORIGIN_REGEX_WHITELIST = ('^http://(.+)\.readthedocs\.org$', '^https://(.+)\.readthedocs\.org$')
# So people can post to their accounts
CORS_ALLOW_CREDENTIALS = True
CORS_ALLOW_HEADERS = (
    'x-requested-with',
    'content-type',
    'accept',
    'origin',
    'authorization',
    'x-csrftoken'
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (
    '%s/readthedocs/templates/' % SITE_ROOT,
)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.contrib.messages.context_processors.messages",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request",
    # Read the Docs processor
    "core.context_processors.readthedocs_processor",
    # allauth specific context processors
    "allauth.account.context_processors.account",
    "allauth.socialaccount.context_processors.socialaccount",
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',
    'django.contrib.messages',

    # third party apps
    'pagination',
    'taggit',
    'djangosecure',
    'guardian',
    'django_gravatar',
    'rest_framework',
    'corsheaders',
    'copyright',

    # Celery bits
    'djcelery',

    # daniellindsleyrocksdahouse
    'haystack',
    'tastypie',

    # our apps
    'readthedocs.bookmarks',
    'readthedocs.projects',
    'readthedocs.builds',
    'readthedocs.comments',
    'readthedocs.core',
    'readthedocs.doc_builder',
    'readthedocs.oauth',
    'readthedocs.redirects',
    'readthedocs.rtd_tests',
    'readthedocs.restapi',
    'readthedocs.privacy',
    'readthedocs.gold',
    'readthedocs.donate',

    # allauth
    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'allauth.socialaccount.providers.github',
    'allauth.socialaccount.providers.bitbucket',
    #'allauth.socialaccount.providers.twitter',
]

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
    'PAGINATE_BY': 10
}

if DEBUG:
    INSTALLED_APPS.append('django_extensions')

CELERY_ALWAYS_EAGER = True
CELERYD_TASK_TIME_LIMIT = 60 * 60  # 60 minutes
CELERY_SEND_TASK_ERROR_EMAILS = False
CELERYD_HIJACK_ROOT_LOGGER = False
# Don't queue a bunch of tasks in the workers
CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_CREATE_MISSING_QUEUES = True

CELERY_DEFAULT_QUEUE = 'celery'
CELERY_QUEUES = (
    Queue('celery', Exchange('celery'), routing_key='celery'),
    Broadcast('build_broadcast_tasks'),
)

DEFAULT_FROM_EMAIL = "no-reply@readthedocs.org"
SERVER_EMAIL = DEFAULT_FROM_EMAIL
SESSION_COOKIE_DOMAIN = 'readthedocs.org'

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
    },
}

# Elasticsearch settings.
ES_HOSTS = ['127.0.0.1:9200']
ES_DEFAULT_NUM_REPLICAS = 0
ES_DEFAULT_NUM_SHARDS = 5

ALLOWED_HOSTS = ['*']

ABSOLUTE_URL_OVERRIDES = {
    'auth.user': lambda o: "/profiles/%s/" % o.username
}

INTERNAL_IPS = ('127.0.0.1',)

backup_count = 1000
maxBytes = 500 * 100 * 100
if LOG_DEBUG:
    backup_count = 2
    maxBytes = 500 * 100 * 10

# Guardian Settings
GUARDIAN_RAISE_403 = True
ANONYMOUS_USER_ID = -1

# Stripe
STRIPE_SECRET = None
STRIPE_PUBLISHABLE = None

# RTD Settings
REPO_LOCK_SECONDS = 30
ALLOW_PRIVATE_REPOS = False

GLOBAL_ANALYTICS_CODE = 'UA-17997319-1'

GRAVATAR_DEFAULT_IMAGE = 'http://media.readthedocs.org/images/silhouette.png'

COPY_START_YEAR = 2010

LOG_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s"

RESTRUCTUREDTEXT_FILTER_SETTINGS = {
    'cloak_email_addresses': True,
    'file_insertion_enabled': False,
    'raw_enabled': False,
    'strip_comments': True,
    'doctitle_xform': True,
    'sectsubtitle_xform': True,
    'initial_header_level': 2,
    'report_level': 5,
    'syntax_highlight': 'none',
    'math_output': 'latex',
    'field_name_limit': 50,
}

LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format': LOG_FORMAT,
            'datefmt': "%d/%b/%Y %H:%M:%S"
        },
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level': 'DEBUG',
            'class': 'django.utils.log.NullHandler',
        },
        'exceptionlog': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "exceptions.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'errorlog': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "rtd.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'postcommit': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "postcommit.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'middleware': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "middleware.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'restapi': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "api.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'db': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "db.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'search': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "search.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler',
        },
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console', 'errorlog'],
            'propagate': True,
            'level': 'WARN',
        },
        'django.db.backends': {
            'handlers': ['db'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'core.views.post_commit': {
            'handlers': ['postcommit'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'core.middleware': {
            'handlers': ['middleware'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'restapi': {
            'handlers': ['restapi'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'django.request': {
            'handlers': ['exceptionlog'],
            'level': 'ERROR',
            'propagate': False,
        },
        'projects.views.public.search': {
            'handlers': ['search'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'search': {
            'handlers': ['search'],
            'level': 'DEBUG',
            'propagate': False,
        },
        # Uncomment if you want to see Elasticsearch queries in the console.
        #'elasticsearch.trace': {
        #    'level': 'DEBUG',
        #    'handlers': ['console'],
        #},

        # Default handler for everything that we're doing. Hopefully this
        # doesn't double-print the Django things as well. Not 100% sure how
        # logging works :)
        '': {
            'handlers': ['console', 'errorlog'],
            'level': 'INFO',
        },
    }
}

if DEBUG:
    LOGGING['handlers']['console']['level'] = 'DEBUG'
<commit_msg>More base config<commit_after># encoding: utf-8
import os
import djcelery
from kombu.common import Broadcast
from kombu import Exchange, Queue

djcelery.setup_loader()

_ = gettext = lambda s: s

DEBUG = True
TEMPLATE_DEBUG = DEBUG
TASTYPIE_FULL_DEBUG = True
LOG_DEBUG = False

PRODUCTION_DOMAIN = 'readthedocs.org'
USE_SUBDOMAIN = False

ADMINS = (
    ('Eric Holscher', 'eric@readthedocs.org'),
    ('Anthony Johnson', 'anthony@readthedocs.org'),
)

MANAGERS = ADMINS

SITE_ROOT = '/'.join(os.path.dirname(os.path.realpath(__file__)).split('/')[0:-2])
DOCROOT = os.path.join(SITE_ROOT, 'user_builds')
UPLOAD_ROOT = os.path.join(SITE_ROOT, 'user_uploads')
CNAME_ROOT = os.path.join(SITE_ROOT, 'cnames')
LOGS_ROOT = os.path.join(SITE_ROOT, 'logs')

# A new base for production files
PRODUCTION_ROOT = os.path.join(SITE_ROOT, 'prod_artifacts')
PRODUCTION_MEDIA_ARTIFACTS = os.path.join(PRODUCTION_ROOT, 'media')

MEDIA_ROOT = '%s/media/' % (SITE_ROOT)
MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/media/admin/'

GROK_API_HOST = 'https://api.grokthedocs.com'

# For 1.4
STATIC_ROOT = os.path.join(SITE_ROOT, 'media/static/')
STATIC_URL = '/static/'
STATICFILES_DIRS = [os.path.join(SITE_ROOT, 'readthedocs', 'static')]
#STATICFILES_FINDERS = ()

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'PREFIX': 'docs',
    }
}

CACHE_MIDDLEWARE_SECONDS = 60

LOGIN_REDIRECT_URL = '/dashboard/'
FORCE_WWW = False
#APPEND_SLASH = False

# Docker
DOCKER_ENABLE = False
DOCKER_IMAGE = 'rtfd-build'

TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
LANGUAGES = (
    ('en', gettext('English')),
    ('es', gettext('Spanish')),
    ('nb', gettext('Norwegian Bokmål')),
    ('fr', gettext('French')),
    ('ru', gettext('Russian')),
    ('de', gettext('German')),
    ('gl', gettext('Galician')),
    ('vi', gettext('Vietnamese')),
    ('zh-cn', gettext('Chinese')),
    ('zh-tw', gettext('Taiwanese')),
    ('ja', gettext('Japanese')),
    ('uk', gettext('Ukrainian')),
    ('it', gettext('Italian')),
)
LOCALE_PATHS = [
    os.path.join(SITE_ROOT, 'readthedocs', 'locale'),
]

USE_I18N = True
USE_L10N = True
SITE_ID = 1
SECRET_KEY = 'replace-this-please'

ACCOUNT_ACTIVATION_DAYS = 7

ATOMIC_REQUESTS = True

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'djangosecure.middleware.SecurityMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'pagination.middleware.PaginationMiddleware',
    # Hack
    # 'core.underscore_middleware.UnderscoreMiddleware',
    'core.middleware.SubdomainMiddleware',
    'core.middleware.SingleVersionMiddleware',
    'corsheaders.middleware.CorsMiddleware',
    #'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
)

AUTHENTICATION_BACKENDS = (
    # Needed to login by username in Django admin, regardless of `allauth`
    "django.contrib.auth.backends.ModelBackend",
    # `allauth` specific authentication methods, such as login by e-mail
    "allauth.account.auth_backends.AuthenticationBackend",
)

# All auth

ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = "mandatory"
ACCOUNT_AUTHENTICATION_METHOD = "username_email"
SOCIALACCOUNT_EMAIL_VERIFICATION = "none"

CORS_ORIGIN_REGEX_WHITELIST = ('^http://(.+)\.readthedocs\.org$', '^https://(.+)\.readthedocs\.org$')
# So people can post to their accounts
CORS_ALLOW_CREDENTIALS = True
CORS_ALLOW_HEADERS = (
    'x-requested-with',
    'content-type',
    'accept',
    'origin',
    'authorization',
    'x-csrftoken'
)

ROOT_URLCONF = 'readthedocs.urls'

TEMPLATE_DIRS = (
    '%s/readthedocs/templates/' % SITE_ROOT,
)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.contrib.messages.context_processors.messages",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request",
    # Read the Docs processor
    "core.context_processors.readthedocs_processor",
    # allauth specific context processors
    "allauth.account.context_processors.account",
    "allauth.socialaccount.context_processors.socialaccount",
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',
    'django.contrib.messages',

    # third party apps
    'pagination',
    'taggit',
    'djangosecure',
    'guardian',
    'django_gravatar',
    'rest_framework',
    'corsheaders',
    'copyright',

    # Celery bits
    'djcelery',

    # daniellindsleyrocksdahouse
    'haystack',
    'tastypie',

    # our apps
    'readthedocs.bookmarks',
    'readthedocs.projects',
    'readthedocs.builds',
    'readthedocs.comments',
    'readthedocs.core',
    'readthedocs.doc_builder',
    'readthedocs.oauth',
    'readthedocs.redirects',
    'readthedocs.rtd_tests',
    'readthedocs.restapi',
    'readthedocs.privacy',
    'readthedocs.gold',
    'readthedocs.donate',

    # allauth
    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'allauth.socialaccount.providers.github',
    'allauth.socialaccount.providers.bitbucket',
    #'allauth.socialaccount.providers.twitter',
]

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
    'PAGINATE_BY': 10
}

if DEBUG:
    INSTALLED_APPS.append('django_extensions')

CELERY_ALWAYS_EAGER = True
CELERYD_TASK_TIME_LIMIT = 60 * 60  # 60 minutes
CELERY_SEND_TASK_ERROR_EMAILS = False
CELERYD_HIJACK_ROOT_LOGGER = False
# Don't queue a bunch of tasks in the workers
CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_CREATE_MISSING_QUEUES = True

CELERY_DEFAULT_QUEUE = 'celery'
CELERY_QUEUES = (
    Queue('celery', Exchange('celery'), routing_key='celery'),
    Broadcast('build_broadcast_tasks'),
)

DEFAULT_FROM_EMAIL = "no-reply@readthedocs.org"
SERVER_EMAIL = DEFAULT_FROM_EMAIL
SESSION_COOKIE_DOMAIN = 'readthedocs.org'

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
    },
}

# Elasticsearch settings.
ES_HOSTS = ['127.0.0.1:9200']
ES_DEFAULT_NUM_REPLICAS = 0
ES_DEFAULT_NUM_SHARDS = 5

ALLOWED_HOSTS = ['*']

ABSOLUTE_URL_OVERRIDES = {
    'auth.user': lambda o: "/profiles/%s/" % o.username
}

INTERNAL_IPS = ('127.0.0.1',)

backup_count = 1000
maxBytes = 500 * 100 * 100
if LOG_DEBUG:
    backup_count = 2
    maxBytes = 500 * 100 * 10

# Guardian Settings
GUARDIAN_RAISE_403 = True
ANONYMOUS_USER_ID = -1

# Stripe
STRIPE_SECRET = None
STRIPE_PUBLISHABLE = None

# RTD Settings
REPO_LOCK_SECONDS = 30
ALLOW_PRIVATE_REPOS = False

GLOBAL_ANALYTICS_CODE = 'UA-17997319-1'

GRAVATAR_DEFAULT_IMAGE = 'http://media.readthedocs.org/images/silhouette.png'

COPY_START_YEAR = 2010

LOG_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s"

RESTRUCTUREDTEXT_FILTER_SETTINGS = {
    'cloak_email_addresses': True,
    'file_insertion_enabled': False,
    'raw_enabled': False,
    'strip_comments': True,
    'doctitle_xform': True,
    'sectsubtitle_xform': True,
    'initial_header_level': 2,
    'report_level': 5,
    'syntax_highlight': 'none',
    'math_output': 'latex',
    'field_name_limit': 50,
}

LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format': LOG_FORMAT,
            'datefmt': "%d/%b/%Y %H:%M:%S"
        },
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level': 'DEBUG',
            'class': 'django.utils.log.NullHandler',
        },
        'exceptionlog': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "exceptions.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'errorlog': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "rtd.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'postcommit': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "postcommit.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'middleware': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "middleware.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'restapi': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "api.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'db': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "db.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'search': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOGS_ROOT, "search.log"),
            'maxBytes': maxBytes,
            'backupCount': backup_count,
            'formatter': 'standard',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler',
        },
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console', 'errorlog'],
            'propagate': True,
            'level': 'WARN',
        },
        'django.db.backends': {
            'handlers': ['db'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'core.views.post_commit': {
            'handlers': ['postcommit'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'core.middleware': {
            'handlers': ['middleware'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'restapi': {
            'handlers': ['restapi'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'django.request': {
            'handlers': ['exceptionlog'],
            'level': 'ERROR',
            'propagate': False,
        },
        'projects.views.public.search': {
            'handlers': ['search'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'search': {
            'handlers': ['search'],
            'level': 'DEBUG',
            'propagate': False,
        },
        # Uncomment if you want to see Elasticsearch queries in the console.
        #'elasticsearch.trace': {
        #    'level': 'DEBUG',
        #    'handlers': ['console'],
        #},

        # Default handler for everything that we're doing. Hopefully this
        # doesn't double-print the Django things as well. Not 100% sure how
        # logging works :)
        '': {
            'handlers': ['console', 'errorlog'],
            'level': 'INFO',
        },
    }
}

if DEBUG:
    LOGGING['handlers']['console']['level'] = 'DEBUG'
<|endoftext|>


<commit_before>try:
    import admin_notifications
    has_admin_notifications = True
except ImportError:
    has_admin_notifications = False

from models import Link

# a global variable, showing whether linkcheck is still working
still_updating = False

def notification():
    if still_updating:
        return "Still checking. Please refresh this page in a short while. "
    else:
        broken_links = Link.objects.filter(ignore=False, url__status=False).count()
        if broken_links:
            return "You have %s broken link%s.
You can view or fix them using the Link Manager." % (broken_links, broken_links>1 and "s" or "")
        else:
            return ''

if has_admin_notifications:
    admin_notifications.register(notification)
<commit_msg>Revert "Make admin_notifications really optional"<commit_after>import admin_notifications

from models import Link

# a global variable, showing whether linkcheck is still working
still_updating = False

def notification():
    if still_updating:
        return "Still checking. Please refresh this page in a short while. "
    else:
        broken_links = Link.objects.filter(ignore=False, url__status=False).count()
        if broken_links:
            return "You have %s broken link%s.
You can view or fix them using the Link Manager." % (broken_links, "s" if broken_links>1 else "")
        else:
            return ''

admin_notifications.register(notification)<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from indico.core.db.sqlalchemy import db, PyIntEnum, UTCDateTime
from indico.util.date_time import now_utc
from indico.util.string import format_repr, return_ascii
from indico.util.struct.enum import IndicoEnum

class AbstractAction(int, IndicoEnum):
    accept = 1
    reject = 2
    change_tracks = 3
    mark_as_duplicate = 4
    merge = 5

class AbstractReview(db.Model):
    """Represents an abstract review, emitted by a reviewer"""

    __tablename__ = 'abstract_reviews'
    __table_args__ = (db.UniqueConstraint('abstract_id', 'user_id', 'track_id'),
                      db.CheckConstraint("(proposed_action = {}) = (proposed_track_id IS NOT NULL)"
                                         .format(AbstractAction.accept), name='prop_track_id_only_accepted'),
                      db.CheckConstraint("proposed_action = {} OR (proposed_contribution_type_id IS NULL)"
                                         .format(AbstractAction.accept), name='prop_contrib_id_only_accepted'),
                      db.CheckConstraint("(proposed_action IN ({}, {})) = (proposed_related_abstract_id IS NOT NULL)"
                                         .format(AbstractAction.mark_as_duplicate, AbstractAction.merge),
                                         name='prop_abstract_id_only_duplicate_merge'),
                      {'schema': 'event_abstracts'})

    id = db.Column(
        db.Integer,
        primary_key=True
    )
    abstract_id = db.Column(
        db.Integer,
        db.ForeignKey('event_abstracts.abstracts.id'),
        index=True,
        nullable=False
    )
    user_id = db.Column(
        db.Integer,
        db.ForeignKey('users.users.id'),
        index=True,
        nullable=False
    )
    track_id = db.Column(
        db.Integer,
        db.ForeignKey('events.tracks.id'),
        index=True,
        nullable=True
    )
    created_dt = db.Column(
        UTCDateTime,
        nullable=False,
        default=now_utc,
    )
    modified_dt = db.Column(
        UTCDateTime,
        nullable=True
    )
    comment = db.Column(
        db.Text,
        nullable=False,
        default=''
    )
    proposed_action = db.Column(
        PyIntEnum(AbstractAction),
        nullable=False
    )
    proposed_related_abstract_id = db.Column(
        db.Integer,
        db.ForeignKey('event_abstracts.abstracts.id'),
        index=True,
        nullable=True
    )
    proposed_track_id = db.Column(
        db.Integer,
        db.ForeignKey('events.tracks.id'),
        index=True,
        nullable=True
    )
    proposed_contribution_type_id = db.Column(
        db.Integer,
        db.ForeignKey('events.contribution_types.id'),
        nullable=True,
        index=True
    )
    abstract = db.relationship(
        'Abstract',
        lazy=True,
        foreign_keys=abstract_id,
        backref=db.backref(
            'reviews',
            cascade='all, delete-orphan',
            lazy=True
        )
    )
    user = db.relationship(
        'User',
        lazy=True,
        backref=db.backref(
            'abstract_reviews',
            lazy='dynamic'
        )
    )
    track = db.relationship(
        'Track',
        lazy=True,
        foreign_keys=track_id,
        backref=db.backref(
            'abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_related_abstract = db.relationship(
        'Abstract',
        lazy=True,
        foreign_keys=proposed_related_abstract_id,
        backref=db.backref(
            'proposed_related_abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_other_tracks = db.relationship(
        'Track',
        secondary='event_abstracts.proposed_for_tracks',
        lazy=True,
        collection_class=set,
        backref=db.backref(
            'proposed_other_abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_track = db.relationship(
        'Track',
        lazy=True,
        foreign_keys=proposed_track_id,
        backref=db.backref(
            'proposed_abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_contribution_type = db.relationship(
        'ContributionType',
        lazy=True,
        backref=db.backref(
            'abstract_reviews',
            lazy='dynamic'
        )
    )

    # relationship backrefs:
    # - ratings (AbstractReviewRating.review)

    @return_ascii
    def __repr__(self):
        return format_repr(self, 'id', 'abstract_id', 'user_id', proposed_action=None)
<commit_msg>Add score property in AbstractReview<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from indico.core.db.sqlalchemy import db, PyIntEnum, UTCDateTime
from indico.util.date_time import now_utc
from indico.util.string import format_repr, return_ascii
from indico.util.struct.enum import IndicoEnum

class AbstractAction(int, IndicoEnum):
    accept = 1
    reject = 2
    change_tracks = 3
    mark_as_duplicate = 4
    merge = 5

class AbstractReview(db.Model):
    """Represents an abstract review, emitted by a reviewer"""

    __tablename__ = 'abstract_reviews'
    __table_args__ = (db.UniqueConstraint('abstract_id', 'user_id', 'track_id'),
                      db.CheckConstraint("(proposed_action = {}) = (proposed_track_id IS NOT NULL)"
                                         .format(AbstractAction.accept), name='prop_track_id_only_accepted'),
                      db.CheckConstraint("proposed_action = {} OR (proposed_contribution_type_id IS NULL)"
                                         .format(AbstractAction.accept), name='prop_contrib_id_only_accepted'),
                      db.CheckConstraint("(proposed_action IN ({}, {})) = (proposed_related_abstract_id IS NOT NULL)"
                                         .format(AbstractAction.mark_as_duplicate, AbstractAction.merge),
                                         name='prop_abstract_id_only_duplicate_merge'),
                      {'schema': 'event_abstracts'})

    id = db.Column(
        db.Integer,
        primary_key=True
    )
    abstract_id = db.Column(
        db.Integer,
        db.ForeignKey('event_abstracts.abstracts.id'),
        index=True,
        nullable=False
    )
    user_id = db.Column(
        db.Integer,
        db.ForeignKey('users.users.id'),
        index=True,
        nullable=False
    )
    track_id = db.Column(
        db.Integer,
        db.ForeignKey('events.tracks.id'),
        index=True,
        nullable=True
    )
    created_dt = db.Column(
        UTCDateTime,
        nullable=False,
        default=now_utc,
    )
    modified_dt = db.Column(
        UTCDateTime,
        nullable=True
    )
    comment = db.Column(
        db.Text,
        nullable=False,
        default=''
    )
    proposed_action = db.Column(
        PyIntEnum(AbstractAction),
        nullable=False
    )
    proposed_related_abstract_id = db.Column(
        db.Integer,
        db.ForeignKey('event_abstracts.abstracts.id'),
        index=True,
        nullable=True
    )
    proposed_track_id = db.Column(
        db.Integer,
        db.ForeignKey('events.tracks.id'),
        index=True,
        nullable=True
    )
    proposed_contribution_type_id = db.Column(
        db.Integer,
        db.ForeignKey('events.contribution_types.id'),
        nullable=True,
        index=True
    )
    abstract = db.relationship(
        'Abstract',
        lazy=True,
        foreign_keys=abstract_id,
        backref=db.backref(
            'reviews',
            cascade='all, delete-orphan',
            lazy=True
        )
    )
    user = db.relationship(
        'User',
        lazy=True,
        backref=db.backref(
            'abstract_reviews',
            lazy='dynamic'
        )
    )
    track = db.relationship(
        'Track',
        lazy=True,
        foreign_keys=track_id,
        backref=db.backref(
            'abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_related_abstract = db.relationship(
        'Abstract',
        lazy=True,
        foreign_keys=proposed_related_abstract_id,
        backref=db.backref(
            'proposed_related_abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_other_tracks = db.relationship(
        'Track',
        secondary='event_abstracts.proposed_for_tracks',
        lazy=True,
        collection_class=set,
        backref=db.backref(
            'proposed_other_abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_track = db.relationship(
        'Track',
        lazy=True,
        foreign_keys=proposed_track_id,
        backref=db.backref(
            'proposed_abstract_reviews',
            lazy='dynamic'
        )
    )
    proposed_contribution_type = db.relationship(
        'ContributionType',
        lazy=True,
        backref=db.backref(
            'abstract_reviews',
            lazy='dynamic'
        )
    )

    # relationship backrefs:
    # - ratings (AbstractReviewRating.review)

    @return_ascii
    def __repr__(self):
        return format_repr(self, 'id', 'abstract_id', 'user_id', proposed_action=None)

    @property
    def score(self):
        if not self.ratings:
            return None
        return sum((x.value for x in self.ratings)) / len(self.ratings)
<|endoftext|>


<commit_before>from toolchain import Recipe, shprint
from os.path import join, exists
import sh
import shutil

class FreetypeRecipe(Recipe):
    version = "2.5.5"
    url = "http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.bz2"
    library = "objs/.libs/libfreetype.a"
    include_dir = ["include", ("builds/unix/ftconfig.h", "config/ftconfig.h")]
    include_per_arch = True

    def build_arch(self, arch):
        build_env = arch.get_env()
        configure = sh.Command(join(self.build_dir, "configure"))
        shprint(configure,
                "CC={}".format(build_env["CC"]),
                "LD={}".format(build_env["LD"]),
                "CFLAGS={}".format(build_env["CFLAGS"]),
                "LDFLAGS={}".format(build_env["LDFLAGS"]),
                "--prefix=/",
                "--host={}".format(arch.triple),
                "--enable-static=yes",
                "--enable-shared=no")
        shprint(sh.make, "clean")
        shprint(sh.make)

recipe = FreetypeRecipe()

<commit_msg>freetype: don't include png/bzip2 support, as we don't have libpng/libbz2 available<commit_after>from toolchain import Recipe, shprint
from os.path import join, exists
import sh
import shutil

class FreetypeRecipe(Recipe):
    version = "2.5.5"
    url = "http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.bz2"
    library = "objs/.libs/libfreetype.a"
    include_dir = ["include", ("builds/unix/ftconfig.h", "config/ftconfig.h")]
    include_per_arch = True

    def build_arch(self, arch):
        build_env = arch.get_env()
        configure = sh.Command(join(self.build_dir, "configure"))
        shprint(configure,
                "CC={}".format(build_env["CC"]),
                "LD={}".format(build_env["LD"]),
                "CFLAGS={}".format(build_env["CFLAGS"]),
                "LDFLAGS={}".format(build_env["LDFLAGS"]),
                "--prefix=/",
                "--host={}".format(arch.triple),
                "--without-png",
                "--without-bzip2",
                "--without-fsspec",
                "--without-old-mac-fonts",
                "--enable-static=yes",
                "--enable-shared=no")
        shprint(sh.make, "clean")
        shprint(sh.make)

recipe = FreetypeRecipe()

<|endoftext|>


<commit_before># Import some necessary libraries.
import sys, time
from server import *
from utils import *
from config import *

class Channel(Server):
    def __init__(self, owner, ircsock, channel, botnick, memorysize, modules):
        super(Channel, self).__init__(ircsock, botnick)
        self.owner = owner
        self.channel = channel
        self.memorysize = memorysize
        self.memory = []
        self.modules = owner.modules
        self.joinchan(self.channel)
        self.plugins = []
        for module in self.modules:
            self.plugins.append(module.Load(self))

    def elephant(self, msg):
        if msg.find("PRIVMSG"):
            try:
                privmsg = msg.split("PRIVMSG " + self.channel + " :")[1]
                usrname = msg.split('!')[0].split(':')[1]
                if len(self.memory) >= memorysize:
                    self.memory.pop(0)
                self.memory.append(usrname +": " +privmsg)
            except:
                pass

    def replay(self, criteria):
        if "last" in criteria.lower():
            line = len(self.memory) - int(criteria.lower().split("last ")[1])
            if line < 0 or line > len(self.memory):
                line = 0
            while line < len(self.memory):
                self.sendmsg(self.memory[line]+"\n", False)
                line += 1
        elif "first" in criteria.lower():
            line = int(criteria.lower().split("first ")[1])
            temp = 0
            while temp < len(self.memory) and temp < line:
                self.sendmsg(self.memory[line]+"\n", False)
                temp += 1
        elif "status" in criteria.lower() or "free" in criteria.lower():
            size = len(self.memory)
            self.sendmsg("Currently memorized " + str(size) + " lines of this conversation." +"\n", False)
            self.sendmsg(str(self.memorysize-size) + " additional slots remaining in buffer." +"\n", False)
        else:
            self.sendmsg("Say '" + self.botnick + "' replay <first/last/status> <numlines>"+"\n", False)

    def read(self, ircmsg):
            # Necessary functions
            if ircmsg.lower().find(self.channel) != -1:
                self.elephant(ircmsg)

            if ircmsg.lower().find(self.channel) != -1 and ircmsg.lower().find(self.botnick) != -1 and ircmsg.find("replay") != -1:
                self.replay(ircmsg.lower().split('replay')[1])

            elif ircmsg.lower().find(self.botnick) != -1 and ircmsg.find("join") != -1:
                newchan = ircmsg.split("join")[1]
                newchan = newchan.strip()
                channels = []
                for channel in self.owner.channels:
                    channels.append(channel.channel)
                if newchan not in channels:
                    self.owner.channels.append(Channel(self.owner, self.ircsock, newchan, self.botnick, self.memorysize, self.modules))

            elif ircmsg.lower().find(self.channel) != -1 and ircmsg.lower().find(self.botnick) != -1 and ircmsg.find("leave") != -1:
                self.sendmsg("Bye!")
                self.leavechan()
                del self.owner.channels[self.owner.channels.index(self)]
                del self

            else:
                for plugin in self.plugins:
                    if plugin.run(ircmsg) == True:
                        break

<commit_msg>Fixed bug in replay, caused crash if lines not set<commit_after># Import some necessary libraries.
import sys, time
from server import *
from utils import *
from config import *

class Channel(Server):
    def __init__(self, owner, ircsock, channel, botnick, memorysize, modules):
        super(Channel, self).__init__(ircsock, botnick)
        self.owner = owner
        self.channel = channel
        self.memorysize = memorysize
        self.memory = []
        self.modules = owner.modules
        self.joinchan(self.channel)
        self.plugins = []
        for module in self.modules:
            self.plugins.append(module.Load(self))

    def elephant(self, msg):
        if msg.find("PRIVMSG"):
            try:
                privmsg = msg.split("PRIVMSG " + self.channel + " :")[1]
                usrname = msg.split('!')[0].split(':')[1]
                if len(self.memory) >= memorysize:
                    self.memory.pop(0)
                self.memory.append(usrname +": " +privmsg)
            except:
                pass

    def replay(self, criteria):
        if "last" in criteria.lower():
            try:
                line = len(self.memory) - int(criteria.lower().split("last ")[1])
            except:
                line = len(self.memory) - 5
            if line < 0 or line > len(self.memory):
                line = 0
            while line < len(self.memory):
                self.sendmsg(self.memory[line]+"\n", False)
                line += 1
        elif "first" in criteria.lower():
            try:
                line = int(criteria.lower().split("first ")[1])
            except:
                line = 5
            temp = 0
            while temp < len(self.memory) and temp < line:
                self.sendmsg(self.memory[line]+"\n", False)
                temp += 1
        elif "status" in criteria.lower() or "free" in criteria.lower():
            size = len(self.memory)
            self.sendmsg("Currently memorized " + str(size) + " lines of this conversation." +"\n", False)
            self.sendmsg(str(self.memorysize-size) + " additional slots remaining in buffer." +"\n", False)
        else:
            self.sendmsg("Say '" + self.botnick + "' replay <first/last/status> <numlines>"+"\n", False)

    def read(self, ircmsg):
            # Necessary functions
            if ircmsg.lower().find(self.channel) != -1:
                self.elephant(ircmsg)

            if ircmsg.lower().find(self.channel) != -1 and ircmsg.lower().find(self.botnick) != -1 and ircmsg.find("replay") != -1:
                self.replay(ircmsg.lower().split('replay')[1])

            elif ircmsg.lower().find(self.botnick) != -1 and ircmsg.find("join") != -1:
                newchan = ircmsg.split("join")[1]
                newchan = newchan.strip()
                channels = []
                for channel in self.owner.channels:
                    channels.append(channel.channel)
                if newchan not in channels:
                    self.owner.channels.append(Channel(self.owner, self.ircsock, newchan, self.botnick, self.memorysize, self.modules))

            elif ircmsg.lower().find(self.channel) != -1 and ircmsg.lower().find(self.botnick) != -1 and ircmsg.find("leave") != -1:
                self.sendmsg("Bye!")
                self.leavechan()
                del self.owner.channels[self.owner.channels.index(self)]
                del self

            else:
                for plugin in self.plugins:
                    if plugin.run(ircmsg) == True:
                        break

<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common code for ADB and Fastboot CLI.

Usage introspects the given class for methods, args, and docs to show the user.

StartCli handles connecting to a device, calling the expected method, and
outputting the results.
"""

import argparse
import io
import inspect
import logging
import re
import sys
import textwrap
import types

from adb import usb_exceptions

class _PortPathAction(argparse.Action):
  def __call__(self, parser, namespace, values, option_string=None):
    setattr(
        namespace, self.dest,
        [int(i) for i in values.replace('/', ',').split(',')])

class PositionalArg(argparse.Action):
  def __call__(self, parser, namespace, values, option_string=None):
    namespace.positional.append(values)

def GetDeviceArguments():
  group = argparse.ArgumentParser('Device', add_help=False)
  group.add_argument(
      '--timeout_ms', default=10000, type=int, metavar='10000',
      help='Timeout in milliseconds.')
  group.add_argument(
      '--port_path', action=_PortPathAction,
      help='USB port path integers (eg 1,2 or 2,1,1)')
  group.add_argument(
      '-s', '--serial',
      help='Device serial to look for (host:port or USB serial)')
  return group

def GetCommonArguments():
  group = argparse.ArgumentParser('Common', add_help=False)
  group.add_argument('--verbose', action='store_true', help='Enable logging')
  return group

def _DocToArgs(doc):
  """Converts a docstring documenting arguments into a dict."""
  offset = None
  in_arg = False
  out = {}
  for l in doc.splitlines():
    if l.strip() == 'Args:':
      in_arg = True
    elif in_arg:
      if not l.strip():
        break
      if offset is None:
        offset = len(l) - len(l.lstrip())
      l = l[offset:]
      if l[0] == ' ':
        out[m.group(1)] += ' ' + l.lstrip()
      else:
        m = re.match(r'^([a-z_]+): (.+)$', l.strip())
        out[m.group(1)] = m.group(2)
  return out

def MakeSubparser(subparsers, parents, method, arguments=None):
  """Returns an argparse subparser to create a 'subcommand' to adb."""
  name = ('-'.join(re.split(r'([A-Z][a-z]+)', method.__name__)[1:-1:2])).lower()
  help = method.__doc__.splitlines()[0]
  subparser = subparsers.add_parser(
      name=name, description=help, help=help.rstrip('.'), parents=parents)
  subparser.set_defaults(method=method, positional=[])
  argspec = inspect.getargspec(method)

  # Figure out positionals and default argument, if any. Explicitly includes
  # arguments that default to '' but excludes arguments that default to None.
  offset = len(argspec.args) - len(argspec.defaults or []) - 1
  positional = []
  for i in xrange(1, len(argspec.args)):
    if i > offset and argspec.defaults[i-offset-1] is None:
      break
    positional.append(argspec.args[i])
  defaults = [None] * offset + list(argspec.defaults or [])

  # Add all arguments so they append to args.positional.
  args_help = _DocToArgs(method.__doc__)
  for name, default in zip(positional, defaults):
    if not isinstance(default, (None.__class__, str)):
      continue
    subparser.add_argument(
        name, help=(arguments or {}).get(name, args_help.get(name)),
        default=default, nargs='?' if default is not None else None,
        action=PositionalArg)
  if argspec.varargs:
    subparser.add_argument(
        argspec.varargs, nargs=argparse.REMAINDER,
        help=(arguments or {}).get(argspec.varargs, args_help.get(argspec.varargs)))
  return subparser

def _RunMethod(dev, args, extra):
  """Runs a method registered via MakeSubparser."""
  logging.info('%s(%s)', args.method.__name__, ', '.join(args.positional))
  result = args.method(dev, *args.positional, **extra)
  if result is not None:
    if isinstance(result, io.StringIO):
      sys.stdout.write(result.getvalue())
    elif isinstance(result, (list, types.GeneratorType)):
      r = ''
      for r in result:
        r = str(r)
        sys.stdout.write(r)
      if not r.endswith('\n'):
        sys.stdout.write('\n')
    else:
      result = str(result)
      sys.stdout.write(result)
      if not result.endswith('\n'):
        sys.stdout.write('\n')
  return 0

def StartCli(args, device_factory, extra=None, **device_kwargs):
  """Starts a common CLI interface for this usb path and protocol."""
  try:
    dev = device_factory(
        port_path=args.port_path, serial=args.serial,
        default_timeout_ms=args.timeout_ms, **device_kwargs)
  except usb_exceptions.DeviceNotFoundError as e:
    print >> sys.stderr, 'No device found: %s' % e
    return 1
  except usb_exceptions.CommonUsbError as e:
    print >> sys.stderr, 'Could not connect to device: %s' % e
    return 1
  try:
    return _RunMethod(dev, args, extra or {})
  except Exception as e:  # pylint: disable=broad-except
    sys.stdout.write(str(e))
    return 1
  finally:
    dev.Close()
<commit_msg>Use range for Python 3 support<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common code for ADB and Fastboot CLI.

Usage introspects the given class for methods, args, and docs to show the user.

StartCli handles connecting to a device, calling the expected method, and
outputting the results.
"""

import argparse
import io
import inspect
import logging
import re
import sys
import textwrap
import types

from adb import usb_exceptions

class _PortPathAction(argparse.Action):
  def __call__(self, parser, namespace, values, option_string=None):
    setattr(
        namespace, self.dest,
        [int(i) for i in values.replace('/', ',').split(',')])

class PositionalArg(argparse.Action):
  def __call__(self, parser, namespace, values, option_string=None):
    namespace.positional.append(values)

def GetDeviceArguments():
  group = argparse.ArgumentParser('Device', add_help=False)
  group.add_argument(
      '--timeout_ms', default=10000, type=int, metavar='10000',
      help='Timeout in milliseconds.')
  group.add_argument(
      '--port_path', action=_PortPathAction,
      help='USB port path integers (eg 1,2 or 2,1,1)')
  group.add_argument(
      '-s', '--serial',
      help='Device serial to look for (host:port or USB serial)')
  return group

def GetCommonArguments():
  group = argparse.ArgumentParser('Common', add_help=False)
  group.add_argument('--verbose', action='store_true', help='Enable logging')
  return group

def _DocToArgs(doc):
  """Converts a docstring documenting arguments into a dict."""
  offset = None
  in_arg = False
  out = {}
  for l in doc.splitlines():
    if l.strip() == 'Args:':
      in_arg = True
    elif in_arg:
      if not l.strip():
        break
      if offset is None:
        offset = len(l) - len(l.lstrip())
      l = l[offset:]
      if l[0] == ' ':
        out[m.group(1)] += ' ' + l.lstrip()
      else:
        m = re.match(r'^([a-z_]+): (.+)$', l.strip())
        out[m.group(1)] = m.group(2)
  return out

def MakeSubparser(subparsers, parents, method, arguments=None):
  """Returns an argparse subparser to create a 'subcommand' to adb."""
  name = ('-'.join(re.split(r'([A-Z][a-z]+)', method.__name__)[1:-1:2])).lower()
  help = method.__doc__.splitlines()[0]
  subparser = subparsers.add_parser(
      name=name, description=help, help=help.rstrip('.'), parents=parents)
  subparser.set_defaults(method=method, positional=[])
  argspec = inspect.getargspec(method)

  # Figure out positionals and default argument, if any. Explicitly includes
  # arguments that default to '' but excludes arguments that default to None.
  offset = len(argspec.args) - len(argspec.defaults or []) - 1
  positional = []
  for i in range(1, len(argspec.args)):
    if i > offset and argspec.defaults[i-offset-1] is None:
      break
    positional.append(argspec.args[i])
  defaults = [None] * offset + list(argspec.defaults or [])

  # Add all arguments so they append to args.positional.
  args_help = _DocToArgs(method.__doc__)
  for name, default in zip(positional, defaults):
    if not isinstance(default, (None.__class__, str)):
      continue
    subparser.add_argument(
        name, help=(arguments or {}).get(name, args_help.get(name)),
        default=default, nargs='?' if default is not None else None,
        action=PositionalArg)
  if argspec.varargs:
    subparser.add_argument(
        argspec.varargs, nargs=argparse.REMAINDER,
        help=(arguments or {}).get(argspec.varargs, args_help.get(argspec.varargs)))
  return subparser

def _RunMethod(dev, args, extra):
  """Runs a method registered via MakeSubparser."""
  logging.info('%s(%s)', args.method.__name__, ', '.join(args.positional))
  result = args.method(dev, *args.positional, **extra)
  if result is not None:
    if isinstance(result, io.StringIO):
      sys.stdout.write(result.getvalue())
    elif isinstance(result, (list, types.GeneratorType)):
      r = ''
      for r in result:
        r = str(r)
        sys.stdout.write(r)
      if not r.endswith('\n'):
        sys.stdout.write('\n')
    else:
      result = str(result)
      sys.stdout.write(result)
      if not result.endswith('\n'):
        sys.stdout.write('\n')
  return 0

def StartCli(args, device_factory, extra=None, **device_kwargs):
  """Starts a common CLI interface for this usb path and protocol."""
  try:
    dev = device_factory(
        port_path=args.port_path, serial=args.serial,
        default_timeout_ms=args.timeout_ms, **device_kwargs)
  except usb_exceptions.DeviceNotFoundError as e:
    print >> sys.stderr, 'No device found: %s' % e
    return 1
  except usb_exceptions.CommonUsbError as e:
    print >> sys.stderr, 'Could not connect to device: %s' % e
    return 1
  try:
    return _RunMethod(dev, args, extra or {})
  except Exception as e:  # pylint: disable=broad-except
    sys.stdout.write(str(e))
    return 1
  finally:
    dev.Close()
<|endoftext|>


<commit_before>
# XXX Going to just start writing then refactor later...
def make_wav_files(bounds_and_files, schedule_list):
    import datetime
    import os
    import wave
    from rsab.listenagain import config, schedule, utils
    if config.has_option('main', 'wavs'):
        output_dir = config.get('main', 'wavs')
    else:
        output_dir = os.getcwd()

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    if not bounds_and_files:
        return None
    wav_reader = wave.open(bounds_and_files[0][2], 'rb')
    output_wav_params = wav_reader.getparams()
    output_wav_params = list(output_wav_params)
    output_wav_params[3] = 0
    output_wav_params = tuple(output_wav_params)
    wav_reader.close()

    frame_rate = output_wav_params[2]
    frame_size = output_wav_params[0] + output_wav_params[1]
    if config.has_option('main', 'framechunksize'):
        frame_chunk = config.getint('main', 'framechunksize')
    else:
        frame_chunk = 65536

    print 'Reading audio data in chunks of', utils.format_large_number(frame_chunk), 'frames'
    print 'Output directory:', output_dir

    output_wav_files = []
    bounds_and_files.sort()
    for details in schedule_list:
        if not details['record']:
            continue
        actual_time_start = utils.apply_padding(details['start'], details.get('pad_start', 0), subtract=True)
        actual_time_start = datetime.datetime.combine(details['date'], actual_time_start)
        actual_time_end = utils.apply_padding(details['end'], details.get('pad_end', 0), subtract=False)
        actual_time_end = datetime.datetime.combine(details['date'], actual_time_end)

        print
        print 'Schedule:', schedule.get_schedule_item_as_string(details)

        output_file_name = os.path.join(output_dir, make_output_file_name(details, 'wav'))
        print 'Output file:', os.path.split(output_file_name)[1]

        wav_writer = wave.open(output_file_name, 'wb')
        wav_writer.setparams(output_wav_params)
        for wav_file_start, wav_file_end, wav_file_path in bounds_and_files:
            if wav_file_start.date() != details['date'] \
            or wav_file_end.date() != details['date'] \
            or wav_file_end <= actual_time_start \
            or wav_file_start >= actual_time_end:
                continue

            wav_reader = wave.open(wav_file_path, 'rb')
            start_frame = end_frame = None
            if wav_file_start < actual_time_start:
                start_frame = (actual_time_start - wav_file_start).seconds * frame_rate
            if wav_file_end > actual_time_end:
                end_frame = (actual_time_end - wav_file_start).seconds * frame_rate
            if start_frame is None:
                start_frame = 0
            if end_frame is None:
                end_frame = wav_reader.getnframes()

            to_read = start_frame
            print 'Skipping', utils.format_large_number(to_read), 'frames in', os.path.split(wav_file_path)[1]
            while to_read > 0:
                data = wav_reader.readframes(min(to_read, frame_chunk))
                if not data:
                    break
                to_read -= (len(data) / frame_size)

            to_read = end_frame - start_frame
            print 'Copying', utils.format_large_number(to_read), 'frames from', os.path.split(wav_file_path)[1]
            while to_read > 0:
                data = wav_reader.readframes(min(to_read, frame_chunk))
                if not data:
                    break
                to_read -= (len(data) / frame_size)
                wav_writer.writeframes(data)
            wav_reader.close()

        wrote_frames = wav_writer.getnframes()
        wav_writer.close()
        if wrote_frames:
            print 'Wrote', utils.format_large_number(wrote_frames), 'frames'
            output_wav_files.append(output_file_name)
        else:
            print 'Wrote no frames; deleting empty file'
            os.unlink(output_file_name)

    return output_wav_files

def make_output_file_name(details, ext, extra=None):
    base_items = [
        details['date'].strftime('%Y%m%d'),
        details['start'].strftime('%H%M%S'),
        details['end'].strftime('%H%M%S'),
        ','.join([details['show']] + (details.get('presenters', []))),
    ]
    if details.get('extra', {}):
        import urllib
        base_items.append(urllib.urlencode(details['extra'], doseq=True))
    if extra:
        base_items.append(extra)
    fname = '_'.join(base_items)
    if ext:
        fname += '.' + ext
    return fname

# XXX Print some things here to say we're doing things...
def encode_file(path, details=None):
    from rsab.listenagain import config, ListenAgainConfigError
    import schedule
    import utils

    import os

    if details is None:
        details = schedule.schedule_from_audio_file_name(path)
    if details is None:
        print 'No details available from path name; skipping:', os.path.split(path)[1]
        return None

    # Get items from encoder section.  Find any called "command".
    commands = []
    if config.has_section('encoder'):
        for key, value in config.items('encoder', raw=True):
            if key == 'command' or key.startswith('command.'):
                if key == 'command':
                    after_first = False
                    n = 0
                else:
                    after_first = True
                    try:
                        n = int(key.split('.')[1])
                    except ValueError:
                        raise ListenAgainConfigError('[encoder] command not of form "command.N"', key)
                commands.append( (after_first, n, value) )

    commands.sort()
    if not commands:
        raise ListenAgainConfigError('No encoding commands defined')

    def escape(s):
        return s.replace('"', '\\"')

    if config.has_option('main', 'mp3s'):
        output_dir = config.get('main', 'mp3s')
    else:
        output_dir = os.getcwd()
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    print 'Encoding', os.path.split(path)[1]

    if config.has_option('encoder', 'artist'):
        artist = config.get('encoder', 'artist')
    else:
        artist = 'Radio St Austell Bay'

    show_name = '%s, %s' % (
        utils.get_show_title_string(details['show'], details.get('presenters', [])),
        details['date'].strftime('%A %d %B %Y'),
    )

    input_file_name = path
    output_file_name = os.path.splitext(os.path.split(path)[1])[0] + '.mp3'
    output_file_path = os.path.join(output_dir, output_file_name)
    option_vars = {
        'artist': escape(artist),
        'title': escape(show_name),
        'year': escape(str(details['date'].year)),
        'input': escape(input_file_name),
        'output': escape(output_file_path),
    }
    if config.has_option('encoder', 'image'):
        option_vars['image'] = config.get('encoder', 'image')

    for dummy, dummy, command in commands:
        print '  -> ', command % option_vars
        pipe = os.popen(command % option_vars)
        exit_status = pipe.close()
        # All commands after the first must operate on the output from the
        # first command.
        option_vars['input'] = option_vars['output']

    print 'file done.'
    print

    return output_file_path

<commit_msg>Shows that end on or after midnight need a date bump at the upper end.<commit_after>
# XXX Going to just start writing then refactor later...
def make_wav_files(bounds_and_files, schedule_list):
    import datetime
    import os
    import wave
    from rsab.listenagain import config, schedule, utils
    if config.has_option('main', 'wavs'):
        output_dir = config.get('main', 'wavs')
    else:
        output_dir = os.getcwd()

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    if not bounds_and_files:
        return None
    wav_reader = wave.open(bounds_and_files[0][2], 'rb')
    output_wav_params = wav_reader.getparams()
    output_wav_params = list(output_wav_params)
    output_wav_params[3] = 0
    output_wav_params = tuple(output_wav_params)
    wav_reader.close()

    frame_rate = output_wav_params[2]
    frame_size = output_wav_params[0] + output_wav_params[1]
    if config.has_option('main', 'framechunksize'):
        frame_chunk = config.getint('main', 'framechunksize')
    else:
        frame_chunk = 65536

    print 'Reading audio data in chunks of', utils.format_large_number(frame_chunk), 'frames'
    print 'Output directory:', output_dir

    output_wav_files = []
    bounds_and_files.sort()
    for details in schedule_list:
        if not details['record']:
            continue
        actual_time_start = utils.apply_padding(details['start'], details.get('pad_start', 0), subtract=True)
        actual_time_start = datetime.datetime.combine(details['date'], actual_time_start)
        actual_time_end = utils.apply_padding(details['end'], details.get('pad_end', 0), subtract=False)
        actual_time_end = datetime.datetime.combine(details['date'], actual_time_end)
        if actual_time_end < actual_time_start:
            new_actual_time_end = actual_time_end + datetime.timedelta(days=1)
            # <6 hours means it could be a show -- else it's a mistake!
            if new_actual_time_end - actual_time_start < datetime.timedelta(hours=6):
                actual_time_end = new_actual_time_end
            del new_actual_time_end

        print
        print 'Schedule:', schedule.get_schedule_item_as_string(details)

        output_file_name = os.path.join(output_dir, make_output_file_name(details, 'wav'))
        print 'Output file:', os.path.split(output_file_name)[1]

        wav_writer = wave.open(output_file_name, 'wb')
        wav_writer.setparams(output_wav_params)
        for wav_file_start, wav_file_end, wav_file_path in bounds_and_files:
            if wav_file_start.date() != details['date'] \
            or wav_file_end.date() != details['date'] \
            or wav_file_end <= actual_time_start \
            or wav_file_start >= actual_time_end:
                continue

            wav_reader = wave.open(wav_file_path, 'rb')
            start_frame = end_frame = None
            if wav_file_start < actual_time_start:
                start_frame = (actual_time_start - wav_file_start).seconds * frame_rate
            if wav_file_end > actual_time_end:
                end_frame = (actual_time_end - wav_file_start).seconds * frame_rate
            if start_frame is None:
                start_frame = 0
            if end_frame is None:
                end_frame = wav_reader.getnframes()

            to_read = start_frame
            print 'Skipping', utils.format_large_number(to_read), 'frames in', os.path.split(wav_file_path)[1]
            while to_read > 0:
                data = wav_reader.readframes(min(to_read, frame_chunk))
                if not data:
                    break
                to_read -= (len(data) / frame_size)

            to_read = end_frame - start_frame
            print 'Copying', utils.format_large_number(to_read), 'frames from', os.path.split(wav_file_path)[1]
            while to_read > 0:
                data = wav_reader.readframes(min(to_read, frame_chunk))
                if not data:
                    break
                to_read -= (len(data) / frame_size)
                wav_writer.writeframes(data)
            wav_reader.close()

        wrote_frames = wav_writer.getnframes()
        wav_writer.close()
        if wrote_frames:
            print 'Wrote', utils.format_large_number(wrote_frames), 'frames'
            output_wav_files.append(output_file_name)
        else:
            print 'Wrote no frames; deleting empty file'
            os.unlink(output_file_name)

    return output_wav_files

def make_output_file_name(details, ext, extra=None):
    base_items = [
        details['date'].strftime('%Y%m%d'),
        details['start'].strftime('%H%M%S'),
        details['end'].strftime('%H%M%S'),
        ','.join([details['show']] + (details.get('presenters', []))),
    ]
    if details.get('extra', {}):
        import urllib
        base_items.append(urllib.urlencode(details['extra'], doseq=True))
    if extra:
        base_items.append(extra)
    fname = '_'.join(base_items)
    if ext:
        fname += '.' + ext
    return fname

# XXX Print some things here to say we're doing things...
def encode_file(path, details=None):
    from rsab.listenagain import config, ListenAgainConfigError
    import schedule
    import utils

    import os

    if details is None:
        details = schedule.schedule_from_audio_file_name(path)
    if details is None:
        print 'No details available from path name; skipping:', os.path.split(path)[1]
        return None

    # Get items from encoder section.  Find any called "command".
    commands = []
    if config.has_section('encoder'):
        for key, value in config.items('encoder', raw=True):
            if key == 'command' or key.startswith('command.'):
                if key == 'command':
                    after_first = False
                    n = 0
                else:
                    after_first = True
                    try:
                        n = int(key.split('.')[1])
                    except ValueError:
                        raise ListenAgainConfigError('[encoder] command not of form "command.N"', key)
                commands.append( (after_first, n, value) )

    commands.sort()
    if not commands:
        raise ListenAgainConfigError('No encoding commands defined')

    def escape(s):
        return s.replace('"', '\\"')

    if config.has_option('main', 'mp3s'):
        output_dir = config.get('main', 'mp3s')
    else:
        output_dir = os.getcwd()
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    print 'Encoding', os.path.split(path)[1]

    if config.has_option('encoder', 'artist'):
        artist = config.get('encoder', 'artist')
    else:
        artist = 'Radio St Austell Bay'

    show_name = '%s, %s' % (
        utils.get_show_title_string(details['show'], details.get('presenters', [])),
        details['date'].strftime('%A %d %B %Y'),
    )

    input_file_name = path
    output_file_name = os.path.splitext(os.path.split(path)[1])[0] + '.mp3'
    output_file_path = os.path.join(output_dir, output_file_name)
    option_vars = {
        'artist': escape(artist),
        'title': escape(show_name),
        'year': escape(str(details['date'].year)),
        'input': escape(input_file_name),
        'output': escape(output_file_path),
    }
    if config.has_option('encoder', 'image'):
        option_vars['image'] = config.get('encoder', 'image')

    for dummy, dummy, command in commands:
        print '  -> ', command % option_vars
        pipe = os.popen(command % option_vars)
        exit_status = pipe.close()
        # All commands after the first must operate on the output from the
        # first command.
        option_vars['input'] = option_vars['output']

    print 'file done.'
    print

    return output_file_path

<|endoftext|>


<commit_before>"""
Utility classes and functions related to computing fairness evaluations

:author: Anastassia Loukina (aloukina@ets.org)
:author: Jeremy Biggs (jbiggs@ets.org)
:author: Nitin Madnani (nmadnani@ets.org)

:date: 07/29/2019
:organization: ETS
"""

import pandas as pd
import pickle
import numpy as np

from os.path import join

import statsmodels.formula.api as smf
from statsmodels.stats.anova import anova_lm

from rsmtool.writer import DataWriter
from rsmtool.container import DataContainer

def convert_to_ordered_category(group_values):
    """
    Convert supplied series to an ordered category
    with levels ordered by category size.
    If multiple categories have the same size,
    the order is determined alphabetically.

    Parameters
    ----------
    group_values: pandas Series

    Returns
    -------
    group_ordered_category: pandas Series
    """

    # get ordered list by size

    # We convert the  value counts to data frame to allow for multilevel sorting.
    # This makes sure that the order is consistent and reproducible across runs
    # when there are more than
    # one group with the maximum number of occurrences.
    df_groups_by_size = pd.DataFrame(group_values.value_counts()).reset_index()
    df_groups_by_size.columns = ['group_name', 'count']
    df_groups_by_size_sorted = df_groups_by_size.sort_values(['count', 'group_name'],
                                                            ascending=[False, True])
    groups_by_size = df_groups_by_size_sorted['group_name'].values

    #convert to category and reorder
    group_category = group_values.astype("category")
    group_ordered_category = group_category.cat.reorder_categories(groups_by_size,
                                                                   ordered=True)
    return group_ordered_category

def get_coefficients(fit,
                     base_category):

    """ Extract estimates, significance and confidence intervals
    for members of the group of interest. The names of the
    predictors are processed to remove the prefix added by Statmodels.
    The name of the base category is added in parenthesis to the Intercept.

    Parameters:
    -----------
    fit: fit
        Linear regression object fitted using statsmodels

    base_category: str
        Name of the group used a reference category when fitting the model

    Returns:
    -------
    df_results: pandas DataFrame
        A dataframe with rows for each category and the following columns:
        - estimate
        - P>[t]
        - Lower and upper boundaries for 95% confidence intervals
    """

    # extract the data we need
    df_results = pd.concat([fit.params, fit.pvalues, fit.conf_int()], axis=1)

    df_results.columns = ['estimate', 'P>[t]', '[0.025', '0.975]']

    # identify the rows we are interested in
    groups = ['Intercept'] + [v for v in df_results.index if 'group' in v]

    df_results = df_results.loc[groups]

    # rename the rows
    df_results.index = [v.split('.')[1].strip(']')
                            if not v == 'Intercept'
                            else 'Intercept ({})'.format(base_category)
                            for v in df_results.index]

    return df_results

def write_fairness_results(fit_dictionary,
                           fairness_container,
                           group,
                           output_dir,
                           experiment_id,
                           file_format):
    """
    Save the results of fairness analysis to disk.

    Parameters
    ----------
    fit_dictionary: dict
        A dictionary of fitted models generated by get_fairness_analysis
    fairness_container: container.DataContainer
        A data container with the results of fairness analysis generated by get_fairness_analysis
    group: str
        The subgroup considered in this analysis
    output_dir: str
        The directory where the results will be saved
    experiment_id: str
        Experiment id
    file_format: str
        File format for data files
    """

    # let's first save model files and summaries
    for model in fit_dictionary:
        fit = fit_dictionary[model]

        ols_file = join(output_dir, '{}_{}_by_{}.ols'.format(experiment_id,
                                                             model,
                                                             group))
        summary_file = join(output_dir, '{}_{}_by_{}_ols_summary.txt'.format(experiment_id,
                                                                             model,
                                                                             group))
        with open(ols_file, 'wb') as olsf, open(summary_file, 'w') as summf:
            pickle.dump(fit, olsf)
            summf.write(str(fit.summary()))

    # Now let's write out the content of the data container
    writer = DataWriter(experiment_id)
    writer.write_experiment_output(output_dir,
                                   fairness_container,
                                   file_format=file_format,
                                   index=True)

def get_fairness_analysis(df,
                          group,
                          system_score_column,
                          human_score_column='sc1'):
    """
    Compute fairness analyses described in Loukina et al. (2019) (https://aclweb.org/anthology/papers/W/W19/W19-4401/)
    The functions computes how muach variance group memebership explains in
    overall score accuracy (osa), overall score difference (osd) and conditional score difference (csd).
    See paper for further detail.

    Parameters
    ----------
    df: pandas DataFrame
        A dataframe containing columns with numeric human scores, numeric system scores
        and a column showing group membership
    group: str
        Name of the column containing group membership
    system_score_column: str
        Name of the column containing system scores
    human_score_column: str
        Name of the column containing human scores

    Returns
    -------
    model_dict: dictionary
        A dictionary with different proposed metrics as keys
        and fitted models as values
    fairness_container: container.DataContainer
        A datacontainer with the following datasets:
            - `estimates_METRICS_by_GROUP` where `GROUP` corresponds to group
            and `METRICS` can be osa, osd and csd
            Estimates for each group computed by the respective models
            - `fairness_metrics_by_GROUP` - a summary of model fits (R2 and p values)
    """

    # compute error and squared error

    df['error'] = df[system_score_column]-df[human_score_column]
    df['SE'] = df['error']**2

    # convert group values to category and reorder them using
    # the largest category as reference

    df['group'] = convert_to_ordered_category(df[group])
    base_group = df['group'].cat.categories[0]
    df['sc1_cat'] = convert_to_ordered_category(df[human_score_column])

    # Overall score accuracy (OSA)
    # Variance in squared error explained by L1

    # fit the model
    osa_model = smf.ols(formula='SE ~ group', data=df)
    osa_fit = osa_model.fit()

    # collect the results
    osa_dict = {'R2': osa_fit.rsquared_adj,
               'sig': osa_fit.f_pvalue}

    osa_results = pd.Series(osa_dict, name='Overall score accuracy')

    df_coefficients_osa = get_coefficients(osa_fit, base_group)

    # Overall score difference (OSD)
    # variance in signed residuals (raw error) explained by L1

    # fit the model
    osd_model = smf.ols(formula='error ~ group', data=df)
    osd_fit = osd_model.fit()

    # collect the results
    osd_dict = {'R2': osd_fit.rsquared_adj,
               'sig': osd_fit.f_pvalue}

    osd_results = pd.Series(osd_dict, name='Overall score difference')

    df_coefficients_osd = get_coefficients(osd_fit, base_group)

    # conditional score difference CSD
    # Variance in score difference conditioned on Native language

    # fit "null" model with human score only
    csd_null_mod = smf.ols(formula='error ~ sc1_cat', data=df)
    csd_null_fit = csd_null_mod.fit()

    # fit model with both human score and group
    csd_mod = smf.ols(formula='error ~ group + sc1_cat', data=df)
    csd_fit = csd_mod.fit()

    # compare the two models using anova_lm
    # we filter warnings for this function because we get
    # runtime warning due to NaNs in the data.
    # these seem to be by design: https://groups.google.com/forum/#!topic/pystatsmodels/-flY0cNnb3k
    np.warnings.filterwarnings('ignore')
    anova_results = anova_lm(csd_null_fit, csd_fit)
    # we reset warnings
    np.warnings.resetwarnings()

    # collect the results. Note that R2 in this case is a difference
    # in R2 between the two models and significance is obtained from anova
    csd_dict = {'R2': csd_fit.rsquared_adj - csd_null_fit.rsquared_adj,
               'sig': anova_results.values[1][-1]}
    csd_results = pd.Series(csd_dict, name='Conditional score difference')

    df_coefficients_csd = get_coefficients(osd_fit, base_group)

    # create a summary table

    df_r2_all = pd.concat([osa_results, osd_results, csd_results], axis=1, sort=True)
    df_r2_all['base_category'] = base_group

    # assemble all datasets into a DataContainer

    datasets = [{'name': 'estimates_osa_by_{}'.format(group),
                 'frame': df_coefficients_osa},
                {'name': 'estimates_osd_by_{}'.format(group),
                 'frame': df_coefficients_osd},
                {'name': 'estimates_csd_by_{}'.format(group),
                 'frame': df_coefficients_csd},
                {'name': 'fairness_metrics_by_{}'.format(group),
                 'frame': df_r2_all}]

    # assemble all models into a dictionary
    model_dict = {'osa': osa_fit,
                  'osd': osd_fit,
                  'csd': csd_fit}

    return model_dict, DataContainer(datasets=datasets)

<commit_msg>Update rsmtool/fairness_utils.py<commit_after>"""
Utility classes and functions related to computing fairness evaluations

:author: Anastassia Loukina (aloukina@ets.org)
:author: Jeremy Biggs (jbiggs@ets.org)
:author: Nitin Madnani (nmadnani@ets.org)

:date: 07/29/2019
:organization: ETS
"""

import pandas as pd
import pickle
import numpy as np

from os.path import join

import statsmodels.formula.api as smf
from statsmodels.stats.anova import anova_lm

from rsmtool.writer import DataWriter
from rsmtool.container import DataContainer

def convert_to_ordered_category(group_values):
    """
    Convert supplied series to an ordered category
    with levels ordered by category size.
    If multiple categories have the same size,
    the order is determined alphabetically.

    Parameters
    ----------
    group_values: pandas Series

    Returns
    -------
    group_ordered_category: pandas Series
    """

    # get ordered list by size

    # We convert the  value counts to data frame to allow for multilevel sorting.
    # This makes sure that the order is consistent and reproducible across runs
    # when there are more than
    # one group with the maximum number of occurrences.
    df_groups_by_size = pd.DataFrame(group_values.value_counts()).reset_index()
    df_groups_by_size.columns = ['group_name', 'count']
    df_groups_by_size_sorted = df_groups_by_size.sort_values(['count', 'group_name'],
                                                            ascending=[False, True])
    groups_by_size = df_groups_by_size_sorted['group_name'].values

    #convert to category and reorder
    group_category = group_values.astype("category")
    group_ordered_category = group_category.cat.reorder_categories(groups_by_size,
                                                                   ordered=True)
    return group_ordered_category

def get_coefficients(fit,
                     base_category):

    """
    Extract estimates, significance and confidence intervals
    for members of the group of interest. The names of the
    predictors are processed to remove the prefix added by Statmodels.
    The name of the base category is added in parenthesis to the Intercept.

    Parameters:
    -----------
    fit: fit
        Linear regression object fitted using statsmodels

    base_category: str
        Name of the group used a reference category when fitting the model

    Returns:
    -------
    df_results: pandas DataFrame
        A dataframe with rows for each category and the following columns:
        - estimate
        - P>[t]
        - Lower and upper boundaries for 95% confidence intervals
    """

    # extract the data we need
    df_results = pd.concat([fit.params, fit.pvalues, fit.conf_int()], axis=1)

    df_results.columns = ['estimate', 'P>[t]', '[0.025', '0.975]']

    # identify the rows we are interested in
    groups = ['Intercept'] + [v for v in df_results.index if 'group' in v]

    df_results = df_results.loc[groups]

    # rename the rows
    df_results.index = [v.split('.')[1].strip(']')
                            if not v == 'Intercept'
                            else 'Intercept ({})'.format(base_category)
                            for v in df_results.index]

    return df_results

def write_fairness_results(fit_dictionary,
                           fairness_container,
                           group,
                           output_dir,
                           experiment_id,
                           file_format):
    """
    Save the results of fairness analysis to disk.

    Parameters
    ----------
    fit_dictionary: dict
        A dictionary of fitted models generated by get_fairness_analysis
    fairness_container: container.DataContainer
        A data container with the results of fairness analysis generated by get_fairness_analysis
    group: str
        The subgroup considered in this analysis
    output_dir: str
        The directory where the results will be saved
    experiment_id: str
        Experiment id
    file_format: str
        File format for data files
    """

    # let's first save model files and summaries
    for model in fit_dictionary:
        fit = fit_dictionary[model]

        ols_file = join(output_dir, '{}_{}_by_{}.ols'.format(experiment_id,
                                                             model,
                                                             group))
        summary_file = join(output_dir, '{}_{}_by_{}_ols_summary.txt'.format(experiment_id,
                                                                             model,
                                                                             group))
        with open(ols_file, 'wb') as olsf, open(summary_file, 'w') as summf:
            pickle.dump(fit, olsf)
            summf.write(str(fit.summary()))

    # Now let's write out the content of the data container
    writer = DataWriter(experiment_id)
    writer.write_experiment_output(output_dir,
                                   fairness_container,
                                   file_format=file_format,
                                   index=True)

def get_fairness_analysis(df,
                          group,
                          system_score_column,
                          human_score_column='sc1'):
    """
    Compute fairness analyses described in Loukina et al. (2019) (https://aclweb.org/anthology/papers/W/W19/W19-4401/)
    The functions computes how muach variance group memebership explains in
    overall score accuracy (osa), overall score difference (osd) and conditional score difference (csd).
    See paper for further detail.

    Parameters
    ----------
    df: pandas DataFrame
        A dataframe containing columns with numeric human scores, numeric system scores
        and a column showing group membership
    group: str
        Name of the column containing group membership
    system_score_column: str
        Name of the column containing system scores
    human_score_column: str
        Name of the column containing human scores

    Returns
    -------
    model_dict: dictionary
        A dictionary with different proposed metrics as keys
        and fitted models as values
    fairness_container: container.DataContainer
        A datacontainer with the following datasets:
            - `estimates_METRICS_by_GROUP` where `GROUP` corresponds to group
            and `METRICS` can be osa, osd and csd
            Estimates for each group computed by the respective models
            - `fairness_metrics_by_GROUP` - a summary of model fits (R2 and p values)
    """

    # compute error and squared error

    df['error'] = df[system_score_column]-df[human_score_column]
    df['SE'] = df['error']**2

    # convert group values to category and reorder them using
    # the largest category as reference

    df['group'] = convert_to_ordered_category(df[group])
    base_group = df['group'].cat.categories[0]
    df['sc1_cat'] = convert_to_ordered_category(df[human_score_column])

    # Overall score accuracy (OSA)
    # Variance in squared error explained by L1

    # fit the model
    osa_model = smf.ols(formula='SE ~ group', data=df)
    osa_fit = osa_model.fit()

    # collect the results
    osa_dict = {'R2': osa_fit.rsquared_adj,
               'sig': osa_fit.f_pvalue}

    osa_results = pd.Series(osa_dict, name='Overall score accuracy')

    df_coefficients_osa = get_coefficients(osa_fit, base_group)

    # Overall score difference (OSD)
    # variance in signed residuals (raw error) explained by L1

    # fit the model
    osd_model = smf.ols(formula='error ~ group', data=df)
    osd_fit = osd_model.fit()

    # collect the results
    osd_dict = {'R2': osd_fit.rsquared_adj,
               'sig': osd_fit.f_pvalue}

    osd_results = pd.Series(osd_dict, name='Overall score difference')

    df_coefficients_osd = get_coefficients(osd_fit, base_group)

    # conditional score difference CSD
    # Variance in score difference conditioned on Native language

    # fit "null" model with human score only
    csd_null_mod = smf.ols(formula='error ~ sc1_cat', data=df)
    csd_null_fit = csd_null_mod.fit()

    # fit model with both human score and group
    csd_mod = smf.ols(formula='error ~ group + sc1_cat', data=df)
    csd_fit = csd_mod.fit()

    # compare the two models using anova_lm
    # we filter warnings for this function because we get
    # runtime warning due to NaNs in the data.
    # these seem to be by design: https://groups.google.com/forum/#!topic/pystatsmodels/-flY0cNnb3k
    np.warnings.filterwarnings('ignore')
    anova_results = anova_lm(csd_null_fit, csd_fit)
    # we reset warnings
    np.warnings.resetwarnings()

    # collect the results. Note that R2 in this case is a difference
    # in R2 between the two models and significance is obtained from anova
    csd_dict = {'R2': csd_fit.rsquared_adj - csd_null_fit.rsquared_adj,
               'sig': anova_results.values[1][-1]}
    csd_results = pd.Series(csd_dict, name='Conditional score difference')

    df_coefficients_csd = get_coefficients(osd_fit, base_group)

    # create a summary table

    df_r2_all = pd.concat([osa_results, osd_results, csd_results], axis=1, sort=True)
    df_r2_all['base_category'] = base_group

    # assemble all datasets into a DataContainer

    datasets = [{'name': 'estimates_osa_by_{}'.format(group),
                 'frame': df_coefficients_osa},
                {'name': 'estimates_osd_by_{}'.format(group),
                 'frame': df_coefficients_osd},
                {'name': 'estimates_csd_by_{}'.format(group),
                 'frame': df_coefficients_csd},
                {'name': 'fairness_metrics_by_{}'.format(group),
                 'frame': df_r2_all}]

    # assemble all models into a dictionary
    model_dict = {'osa': osa_fit,
                  'osd': osd_fit,
                  'csd': csd_fit}

    return model_dict, DataContainer(datasets=datasets)

<|endoftext|>


<commit_before>import os
import sys
import shutil
import pygit2

SSH_KEY_PUBLIC = os.path.expanduser('~/.ssh/id_rsa.pub')
SSH_KEY_PRIVATE = os.path.expanduser('~/.ssh/id_rsa_unencrypted')
CREDENTIALS = pygit2.credentials.Keypair('git',
                                         SSH_KEY_PUBLIC,
                                         SSH_KEY_PRIVATE,
                                         None)
REMOTE_REPO = 'remote_repo'
LOCAL_REPO = 'local_repo'

# Cleanup
if os.path.exists(LOCAL_REPO):
    shutil.rmtree(LOCAL_REPO)

if os.path.exists(REMOTE_REPO):
    shutil.rmtree(REMOTE_REPO)

version = 1
def create_commits(repo, how_many):
    if repo.head_is_unborn:
        parent = []
    else:
        parent = [repo.head.target]

    global version
    for i in range(how_many):
        test_fp = open(os.path.join(repo.workdir,
                                    os.path.basename(os.path.normpath(repo.workdir)) + '_test.txt'), 'a+')
        test_fp.write('Version %d.\n\n' % (version))
        test_fp.close()
        repo.index.add_all()

        user = repo.default_signature
        tree = repo.index.write_tree()
        commit = repo.create_commit('HEAD',
                                    user,
                                    user,
                                    'Version %d of test.txt on %s' % (version, os.path.basename(os.path.normpath(repo.workdir))),
                                    tree,
                                    parent)
        parent = [commit]
        version += 1

# Repo init
# Initialize new remote repo
remote_repo = pygit2.init_repository(REMOTE_REPO, False)
create_commits(remote_repo, 1)

# Clone local repo
local_repo = pygit2.clone_repository(REMOTE_REPO,
                                     LOCAL_REPO)

# Repo pull fastforwardable
create_commits(remote_repo, 1)
pull(local_repo)

# Repo pull merge necessary
# create_commits(local_repo, 1)
# create_commits(remote_repo, 1)

# pull(local_repo)

# Repo push

# Create commit

# Export files

# Merge Conficts?
<commit_msg>Adding basic pull functionality.<commit_after>import os
import sys
import shutil
import pygit2

SSH_KEY_PUBLIC = os.path.expanduser('~/.ssh/id_rsa.pub')
SSH_KEY_PRIVATE = os.path.expanduser('~/.ssh/id_rsa_unencrypted')
CREDENTIALS = pygit2.credentials.Keypair('git',
                                         SSH_KEY_PUBLIC,
                                         SSH_KEY_PRIVATE,
                                         None)
REMOTE_REPO = 'remote_repo'
LOCAL_REPO = 'local_repo'

# Cleanup
if os.path.exists(LOCAL_REPO):
    shutil.rmtree(LOCAL_REPO)

if os.path.exists(REMOTE_REPO):
    shutil.rmtree(REMOTE_REPO)

version = 1
def create_commits(repo, how_many):
    if repo.head_is_unborn:
        parent = []
    else:
        parent = [repo.head.target]

    global version
    for i in range(how_many):
        test_fp = open(os.path.join(repo.workdir,
                                    os.path.basename(os.path.normpath(repo.workdir)) + '_test.txt'), 'a+')
        print 'Writing %s to %s on %s repo' % ('Version %d.\n' % (version), test_fp.name, repo)
        test_fp.write('Version %d.\n\n' % (version))
        test_fp.close()
        repo.index.add_all()

        user = repo.default_signature
        tree = repo.index.write_tree()
        commit = repo.create_commit('HEAD',
                                    user,
                                    user,
                                    'Version %d of test.txt on %s' % (version, os.path.basename(os.path.normpath(repo.workdir))),
                                    tree,
                                    parent)
        parent = [commit]
        version += 1

def pull(repo, remote_name='origin'):
    for remote in repo.remotes:
        if remote.name == remote_name:
            remote.fetch()
            remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
            merge_result, _ = repo.merge_analysis(remote_master_id)
            # Up to date, do nothing
            if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
                return
            # We can just fastforward
            elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
                print repo.head.target
                print repo.status()
                master_ref = repo.lookup_reference('refs/heads/master')
                master_ref.set_target(remote_master_id)
                repo.head.set_target(remote_master_id)
                repo.checkout_head()
                print repo.status()
            elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
                repo.merge(remote_master_id)
                print repo.index.conflicts

                assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
                user = repo.default_signature
                tree = repo.index.write_tree()
                commit = repo.create_commit('HEAD',
                                            user,
                                            user,
                                            'Merge!',
                                            tree,
                                            [repo.head.target, remote_master_id])
                repo.state_cleanup()
            else:
                raise AssertionError('Unknown merge analysis result')

# Initialize new remote repo
remote_repo = pygit2.init_repository(REMOTE_REPO, False)
create_commits(remote_repo, 1)

# Clone local repo
local_repo = pygit2.clone_repository(REMOTE_REPO,
                                     LOCAL_REPO)

# Repo pull fastforwardable
create_commits(remote_repo, 1)
pull(local_repo)

# Repo pull merge necessary
# create_commits(local_repo, 1)
# create_commits(remote_repo, 1)

# pull(local_repo)

# Repo push

# Create commit

# Export files

# Merge Conficts?
<|endoftext|>