<commit_before># pylint: disable=too-many-ancestors
"""
Views for microsites api end points.
"""
import cStringIO
from collections import OrderedDict
import copy
import logging
from PIL import Image
import sys, yaml
import urllib2
import urlparse

from django.apps import apps
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.files.base import ContentFile, File
from django.db import IntegrityError
from django.shortcuts import get_object_or_404

from edx_rest_framework_extensions.authentication import JwtAuthentication

from rest_framework.response import Response
from rest_framework import status
from rest_framework.authentication import SessionAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework_oauth.authentication import OAuth2Authentication
from rest_framework.viewsets import ViewSet

from microsite_configuration.models import \
    Microsite, MicrositeOrganizationMapping

from microsite_configuration.serializers import \
    MicrositeSerializer, MicrositeModelSerializer

from organizations import api as organizations_api
from organizations import exceptions as org_exceptions
from organizations.models import Organization

from openedx.core.djangoapps.user_api.errors import \
    UserNotFound, UserNotAuthorized

log = logging.getLogger(__name__)

def download_image(url):
    """
    Downloads an image
    Returns a PIL Image, otherwise raises an exception.
    """
    r = urllib2.Request(url)
    try:
        request = urllib2.urlopen(r, timeout=10)
        image_data = cStringIO.StringIO(request.read()) # StringIO is a file
        img = Image.open(image_data) # Creates an instance of PIL Image class
        if img.format in ('PNG', 'png'):
            return img
        else:
            return None
    except urllib2.HTTPError as e:
        log.error(e)
        raise

def save_org_logo(url, org_short_name):
    """
    Download and save remote organization logo
    """
    try:
        image = download_image(url)
        filename = urlparse.urlparse(url).path.split('/')[-1]
        tempfile = image
        tempfile_io = cStringIO.StringIO() # Creates file-like object in mem
        tempfile.save(tempfile_io, format=image.format)
        org = Organization.objects.get(short_name=org_short_name)
        org.logo.save(
            filename,
            ContentFile(tempfile_io.getvalue()),
            save=False
        )
        org.save()
    except Exception as e:
        log.error(e)
        raise

def generate_error_response(string):
    """
    Generate Response with error message about missing request data
    """
    print "in the error fucntion"
    return Response(
        {"error": "{} is None, please add to request body.".format(string)},
        status=status.HTTP_400_BAD_REQUEST
    )

"""
    The following 3 methods are helper functions to:
        1: generate a proper json object for the microsite values
        2: generate a yaml file that gets saved on the server that will be used
           to generate config files for the microsites to run through lms and
           for the nginx site configurations.
        The file location is controlled via EDXAPP_MICROSITE_CONFIG_FILE kept
        in server_vars.yml
"""

def update_map( map, key, values):
    """
    Updated the given map with the key:value pair provided
    """
    if isinstance(values, OrderedDict):
        new_map = {}
        for i in values:
            update_map(new_map, i, values[i])
        map.update({key:new_map})
    else:
        map.update({key:values})

def build_inner_map(microsite):
    """
    simple utility to serve as intermediate between save_to_file and
    update_map to allow for the transformation of a single microsite value set.
    """
    inner_map = {}

    for item in microsite:
        update_map(inner_map, item, microsite[item])
    return inner_map

def save_to_file():
    """
    get all microsites from the database and loop over them contructing a
    dictionary by the use of build_inner_map, each microsite is parsed with
    build_inner_map and then added to a main "outer" dictionary for compiling
    of the yaml file. After all microsites have been parsed the main dictonary
    is dumped to a yaml file.
    """
    microsites = Microsite.objects.all()
    data = {}
    outer_map = {}
    for microsite in microsites:
        outer_map.update({microsite.key: build_inner_map(microsite.values)})

    data.update({"PLATFORM_EDXAPP_MICROSITE_CONFIGURATION":outer_map})

    f = open(settings.MICROSITE_CONFIG_FILE, 'w+')
    f.write("---\n")
    yaml.safe_dump(
        data,
        f,
        default_flow_style=False,
        encoding='utf-8',
        allow_unicode=True
    )
    f.close()

class MicrositesViewSet(ViewSet):
    """
        **Use Cases**
            Microsite view to fetch microsite data and create a new microsite

        **Example Requests**:
            GET /api/microsites/v1/microsites/
            POST /api/microsites/v1/microsites/

        **Response Values for GET**
            If the user is logged in and enrolled, the response contains:
            List of all Microsites
            {
                id:  int
                key: ORG
                values: dict
                site: id
            }

        **Response Values for POST**

            This creates a new Microsite
            request should have the following structure

            {
            "domain_prefix": "staging.ttro",
            "SITE_NAME": "staging.ttro.proversity.io",
            "platform_name": "TTRO",
            "course_org_filter": "TTR",
            "show_homepage_promo_video": false,
            "course_about_show_social_links": false,
            "css_overrides_file": "TTR/css/overrides.css",
            "favicon_path": "TTR/images/favicon.ico",
            "ENABLE_MKTG_URLS": false
            }

         Inserts new organization into app/local state given the following:

         {
             'name': string,
             'short_name': string,
             'description': string,
             'logo': (not required),
         }
         Returns updated dictionary including a new 'id': integer field/value
    """

    queryset = Microsite.objects.all()
    serializer_class = MicrositeSerializer
    lookup_field = 'key'
    authentication_classes =\
        (OAuth2Authentication, JwtAuthentication, SessionAuthentication)

    permission_classes = (IsAuthenticated,)

    def get(self, request, format=None):
        microsites = Microsite.objects.all()

        for microsite in microsites:
            microsite.values = build_inner_map(microsite.values)

        serializer = MicrositeModelSerializer(microsites, many=True)
        return Response(serializer.data)

    def post(self, request, format=None):

        serializer = MicrositeSerializer(data=request.data)
        Site = apps.get_model('sites', 'Site')

        # Parse the request.data as json
        site_url = request.data.get('SITE_NAME', None)
        site_name = request.data.get('domain_prefix', None)
        platform_name = request.data.get('platform_name', None)
        course_org_filter = request.data.get('course_org_filter', None)

        # Check if staging is in the site name, strip staging from the name
        if site_name is not None and 'staging' in site_name:
            site_name = site_name.replace('staging.', '')

        # need to check if site exists, do not duplicate

        try:
            Site.objects.filter(domain=site_url)
            site = Site(domain=site_url, name=site_name)
            site.save()
        except IntegrityError as e:
            return Response({"error": str(e)}, status=status.HTTP_400_BAD_REQUEST)
        # else:
        #     return Response(
        #         {'error': 'That site url already exists'},
        #         status=status.HTTP_400_BAD_REQUEST)

        org_data = {
            'name': platform_name,
            'short_name': course_org_filter,
            'description': platform_name,
        }

        messages = {}

        # Check if org exits, if not add organization
        try:
            organizations_api.get_organization_by_short_name(course_org_filter)
        except org_exceptions.InvalidOrganizationException:
            organizations_api.add_organization(org_data)
        if 's3_logo_url' in request.data:
            s3_logo_url = request.data.get('s3_logo_url', None)
            try:
                save_org_logo(s3_logo_url, course_org_filter)
                messages['logo-image-error'] = "No error."
            except Exception as e:
                messages['logo-image-error'] = '{}'.format(e)
        # Need to check if the microsite key is a duplicate
        try:
            microsite = Microsite(
                key=org_data['short_name'],
                values=request.data,
                site=site
            )
            microsite.save()
        except IntegrityError:
            return Response(
                { "error": "Duplicate entry for microsite key {key}".format(
                    key=org_data['short_name'])
                },
                status=status.HTTP_400_BAD_REQUEST
            )

        # Map the organization and microsite
        mapping = MicrositeOrganizationMapping(
            organization=org_data['short_name'],
            microsite=microsite
        )
        mapping.save()

        if serializer.is_valid():
            serializer.save()
            save_to_file()
            messages['id'] = microsite.id
            return Response(messages, status=status.HTTP_201_CREATED)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class MicrositesDetailView(ViewSet):
    '''
        **Use Cases**
            Microsite view to fetch microsite and update an existing microsite
        **Example Requests**:
            GET /api/microsites/v1/microsites/{pk}
            PUT /api/microsites/v1/microsites/{pk}

        **Response Values for GET**
            If the user is logged in the response contains:
            The microsite
            {
                id:  {pk}
                key: ORG
                values: dict
                site: id
            }

         **Response Values for PUT**
            This updates an existing Microsite
            request.data received from consola has the following structure

            {
                "domain_prefix": "staging.ttro",
                "SITE_NAME": "staging.ttro.proversity.io",
                "platform_name": "TTRO",
                "course_org_filter": "TTR",
                "show_homepage_promo_video": false,
                "course_about_show_social_links": false,
                "css_overrides_file": "TTR/css/overrides.css",
                "favicon_path": "TTR/images/favicon.ico",
                "ENABLE_MKTG_URLS": false
            }
    '''

    authentication_classes =\
        (OAuth2Authentication, JwtAuthentication, SessionAuthentication)

    permission_classes = (IsAuthenticated,)

    def get(self, request, pk, format=None):
        try:
            queryset = Microsite.objects.all()
            microsite = get_object_or_404(queryset, pk=pk)
            microsite.values = build_inner_map(microsite.values)
            serializer = MicrositeModelSerializer(microsite)
        except UserNotAuthorized:
            return Response(status=status.HTTP_403_FORBIDDEN)
        except UserNotFound:
            return Response(
                {"error": "User not found."},
                status=status.HTTP_404_NOT_FOUND
            )

        return Response(serializer.data)

    def delete(self, request, pk, format=None):

        # Get the mircosite
        microsite = Microsite.objects.get(pk=pk)
        microsite_id = {"id": microsite.id}
        domain = microsite.site.domain
        associated_site = Site.objects.get(domain=domain)

        # Delete the site which deletes the microsite and org mapping
        associated_site.delete()

        return Response(
            microsite_id,
            status=status.HTTP_200_OK
        )

    def put(self, request, pk, format=None):

        messages = {}

        # Parse the request.data as json
        site_url = request.data.get('SITE_NAME', None)
        site_name = request.data.get('domain_prefix', None)
        platform_name = request.data.get('platform_name', None)
        course_org_filter = request.data.get('course_org_filter', None)

        # Get the microsite
        microsite = Microsite.objects.get(pk=pk)
        domain = microsite.site.domain
        serializer = MicrositeSerializer(data=request.data)
        if microsite is None:
            return Response(
                {"error": "The microsite was not found"},
                status=status.HTTP_404_NOT_FOUND
            )
        elif microsite.key != course_org_filter:
            return Response(
                {"error": "The course_org_filter cannot be changed."},
                status=status.HTTP_400_BAD_REQUEST
            )

        # If new site url, we need to create a new site and update the mapping
        elif microsite.site.domain != site_url:
            if not Site.objects.filter(domain=site_url):
                new_site = Site(domain=site_url, name=site_name)
                new_site.save()
                Microsite.objects.filter(pk=pk).update(site=new_site)
                old_site = Site.objects.filter(domain=domain)
                old_site.delete()
            else:
                return Response(
                    {'error': 'That site url already exists'},
                    status=status.HTTP_400_BAD_REQUEST
                )

        Microsite.objects.filter(pk=pk).update(values=request.data)

        if 's3_logo_url' in request.data:
            s3_logo_url = request.data.get('s3_logo_url', None)
            try:
                save_org_logo(s3_logo_url, course_org_filter)
                messages['logo-image-error'] = "No error."
            except Exception as e:
                messages['logo-image-error'] = '{}'.format(e)

        if serializer.is_valid():
            serializer.save()
            messages['id'] = microsite.id
            save_to_file()
            return Response(
                messages,
                status=status.HTTP_200_OK
            )
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)<commit_msg>just wrap everything in a try except<commit_after># pylint: disable=too-many-ancestors
"""
Views for microsites api end points.
"""
import cStringIO
from collections import OrderedDict
import copy
import logging
from PIL import Image
import sys, yaml
import urllib2
import urlparse

from django.apps import apps
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.files.base import ContentFile, File
from django.db import IntegrityError
from django.shortcuts import get_object_or_404

from edx_rest_framework_extensions.authentication import JwtAuthentication

from rest_framework.response import Response
from rest_framework import status
from rest_framework.authentication import SessionAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework_oauth.authentication import OAuth2Authentication
from rest_framework.viewsets import ViewSet

from microsite_configuration.models import \
    Microsite, MicrositeOrganizationMapping

from microsite_configuration.serializers import \
    MicrositeSerializer, MicrositeModelSerializer

from organizations import api as organizations_api
from organizations import exceptions as org_exceptions
from organizations.models import Organization

from openedx.core.djangoapps.user_api.errors import \
    UserNotFound, UserNotAuthorized

log = logging.getLogger(__name__)

def download_image(url):
    """
    Downloads an image
    Returns a PIL Image, otherwise raises an exception.
    """
    r = urllib2.Request(url)
    try:
        request = urllib2.urlopen(r, timeout=10)
        image_data = cStringIO.StringIO(request.read()) # StringIO is a file
        img = Image.open(image_data) # Creates an instance of PIL Image class
        if img.format in ('PNG', 'png'):
            return img
        else:
            return None
    except urllib2.HTTPError as e:
        log.error(e)
        raise

def save_org_logo(url, org_short_name):
    """
    Download and save remote organization logo
    """
    try:
        image = download_image(url)
        filename = urlparse.urlparse(url).path.split('/')[-1]
        tempfile = image
        tempfile_io = cStringIO.StringIO() # Creates file-like object in mem
        tempfile.save(tempfile_io, format=image.format)
        org = Organization.objects.get(short_name=org_short_name)
        org.logo.save(
            filename,
            ContentFile(tempfile_io.getvalue()),
            save=False
        )
        org.save()
    except Exception as e:
        log.error(e)
        raise

def generate_error_response(string):
    """
    Generate Response with error message about missing request data
    """
    print "in the error fucntion"
    return Response(
        {"error": "{} is None, please add to request body.".format(string)},
        status=status.HTTP_400_BAD_REQUEST
    )

"""
    The following 3 methods are helper functions to:
        1: generate a proper json object for the microsite values
        2: generate a yaml file that gets saved on the server that will be used
           to generate config files for the microsites to run through lms and
           for the nginx site configurations.
        The file location is controlled via EDXAPP_MICROSITE_CONFIG_FILE kept
        in server_vars.yml
"""

def update_map( map, key, values):
    """
    Updated the given map with the key:value pair provided
    """
    if isinstance(values, OrderedDict):
        new_map = {}
        for i in values:
            update_map(new_map, i, values[i])
        map.update({key:new_map})
    else:
        map.update({key:values})

def build_inner_map(microsite):
    """
    simple utility to serve as intermediate between save_to_file and
    update_map to allow for the transformation of a single microsite value set.
    """
    inner_map = {}

    for item in microsite:
        update_map(inner_map, item, microsite[item])
    return inner_map

def save_to_file():
    """
    get all microsites from the database and loop over them contructing a
    dictionary by the use of build_inner_map, each microsite is parsed with
    build_inner_map and then added to a main "outer" dictionary for compiling
    of the yaml file. After all microsites have been parsed the main dictonary
    is dumped to a yaml file.
    """
    microsites = Microsite.objects.all()
    data = {}
    outer_map = {}
    for microsite in microsites:
        outer_map.update({microsite.key: build_inner_map(microsite.values)})

    data.update({"PLATFORM_EDXAPP_MICROSITE_CONFIGURATION":outer_map})

    f = open(settings.MICROSITE_CONFIG_FILE, 'w+')
    f.write("---\n")
    yaml.safe_dump(
        data,
        f,
        default_flow_style=False,
        encoding='utf-8',
        allow_unicode=True
    )
    f.close()

class MicrositesViewSet(ViewSet):
    """
        **Use Cases**
            Microsite view to fetch microsite data and create a new microsite

        **Example Requests**:
            GET /api/microsites/v1/microsites/
            POST /api/microsites/v1/microsites/

        **Response Values for GET**
            If the user is logged in and enrolled, the response contains:
            List of all Microsites
            {
                id:  int
                key: ORG
                values: dict
                site: id
            }

        **Response Values for POST**

            This creates a new Microsite
            request should have the following structure

            {
            "domain_prefix": "staging.ttro",
            "SITE_NAME": "staging.ttro.proversity.io",
            "platform_name": "TTRO",
            "course_org_filter": "TTR",
            "show_homepage_promo_video": false,
            "course_about_show_social_links": false,
            "css_overrides_file": "TTR/css/overrides.css",
            "favicon_path": "TTR/images/favicon.ico",
            "ENABLE_MKTG_URLS": false
            }

         Inserts new organization into app/local state given the following:

         {
             'name': string,
             'short_name': string,
             'description': string,
             'logo': (not required),
         }
         Returns updated dictionary including a new 'id': integer field/value
    """

    queryset = Microsite.objects.all()
    serializer_class = MicrositeSerializer
    lookup_field = 'key'
    authentication_classes =\
        (OAuth2Authentication, JwtAuthentication, SessionAuthentication)

    permission_classes = (IsAuthenticated,)

    def get(self, request, format=None):
        microsites = Microsite.objects.all()

        for microsite in microsites:
            microsite.values = build_inner_map(microsite.values)

        serializer = MicrositeModelSerializer(microsites, many=True)
        return Response(serializer.data)

    def post(self, request, format=None):

        serializer = MicrositeSerializer(data=request.data)
        Site = apps.get_model('sites', 'Site')

        # Parse the request.data as json
        try:
            site_url = request.data.get('SITE_NAME', None)
            site_name = request.data.get('domain_prefix', None)
            platform_name = request.data.get('platform_name', None)
            course_org_filter = request.data.get('course_org_filter', None)
        # Check if staging is in the site name, strip staging from the name
            if 'staging' in site_name:
                site_name = site_name.replace('staging.', '')
        # need to check if site exists, do not duplicate
            Site.objects.filter(domain=site_url)
            site = Site(domain=site_url, name=site_name)
            site.save()
        except IntegrityError as e:
            return Response({"error": str(e)}, status=status.HTTP_400_BAD_REQUEST)
             status=status.HTTP_400_BAD_REQUEST)

        org_data = {
            'name': platform_name,
            'short_name': course_org_filter,
            'description': platform_name,
        }

        messages = {}

        # Check if org exits, if not add organization
        try:
            organizations_api.get_organization_by_short_name(course_org_filter)
        except org_exceptions.InvalidOrganizationException:
            organizations_api.add_organization(org_data)
        if 's3_logo_url' in request.data:
            s3_logo_url = request.data.get('s3_logo_url', None)
            try:
                save_org_logo(s3_logo_url, course_org_filter)
                messages['logo-image-error'] = "No error."
            except Exception as e:
                messages['logo-image-error'] = '{}'.format(e)
        # Need to check if the microsite key is a duplicate
        try:
            microsite = Microsite(
                key=org_data['short_name'],
                values=request.data,
                site=site
            )
            microsite.save()
        except IntegrityError:
            return Response(
                { "error": "Duplicate entry for microsite key {key}".format(
                    key=org_data['short_name'])
                },
                status=status.HTTP_400_BAD_REQUEST
            )

        # Map the organization and microsite
        mapping = MicrositeOrganizationMapping(
            organization=org_data['short_name'],
            microsite=microsite
        )
        mapping.save()

        if serializer.is_valid():
            serializer.save()
            save_to_file()
            messages['id'] = microsite.id
            return Response(messages, status=status.HTTP_201_CREATED)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class MicrositesDetailView(ViewSet):
    '''
        **Use Cases**
            Microsite view to fetch microsite and update an existing microsite
        **Example Requests**:
            GET /api/microsites/v1/microsites/{pk}
            PUT /api/microsites/v1/microsites/{pk}

        **Response Values for GET**
            If the user is logged in the response contains:
            The microsite
            {
                id:  {pk}
                key: ORG
                values: dict
                site: id
            }

         **Response Values for PUT**
            This updates an existing Microsite
            request.data received from consola has the following structure

            {
                "domain_prefix": "staging.ttro",
                "SITE_NAME": "staging.ttro.proversity.io",
                "platform_name": "TTRO",
                "course_org_filter": "TTR",
                "show_homepage_promo_video": false,
                "course_about_show_social_links": false,
                "css_overrides_file": "TTR/css/overrides.css",
                "favicon_path": "TTR/images/favicon.ico",
                "ENABLE_MKTG_URLS": false
            }
    '''

    authentication_classes =\
        (OAuth2Authentication, JwtAuthentication, SessionAuthentication)

    permission_classes = (IsAuthenticated,)

    def get(self, request, pk, format=None):
        try:
            queryset = Microsite.objects.all()
            microsite = get_object_or_404(queryset, pk=pk)
            microsite.values = build_inner_map(microsite.values)
            serializer = MicrositeModelSerializer(microsite)
        except UserNotAuthorized:
            return Response(status=status.HTTP_403_FORBIDDEN)
        except UserNotFound:
            return Response(
                {"error": "User not found."},
                status=status.HTTP_404_NOT_FOUND
            )

        return Response(serializer.data)

    def delete(self, request, pk, format=None):

        # Get the mircosite
        microsite = Microsite.objects.get(pk=pk)
        microsite_id = {"id": microsite.id}
        domain = microsite.site.domain
        associated_site = Site.objects.get(domain=domain)

        # Delete the site which deletes the microsite and org mapping
        associated_site.delete()

        return Response(
            microsite_id,
            status=status.HTTP_200_OK
        )

    def put(self, request, pk, format=None):

        messages = {}

        # Parse the request.data as json
        site_url = request.data.get('SITE_NAME', None)
        site_name = request.data.get('domain_prefix', None)
        platform_name = request.data.get('platform_name', None)
        course_org_filter = request.data.get('course_org_filter', None)

        # Get the microsite
        microsite = Microsite.objects.get(pk=pk)
        domain = microsite.site.domain
        serializer = MicrositeSerializer(data=request.data)
        if microsite is None:
            return Response(
                {"error": "The microsite was not found"},
                status=status.HTTP_404_NOT_FOUND
            )
        elif microsite.key != course_org_filter:
            return Response(
                {"error": "The course_org_filter cannot be changed."},
                status=status.HTTP_400_BAD_REQUEST
            )

        # If new site url, we need to create a new site and update the mapping
        elif microsite.site.domain != site_url:
            if not Site.objects.filter(domain=site_url):
                new_site = Site(domain=site_url, name=site_name)
                new_site.save()
                Microsite.objects.filter(pk=pk).update(site=new_site)
                old_site = Site.objects.filter(domain=domain)
                old_site.delete()
            else:
                return Response(
                    {'error': 'That site url already exists'},
                    status=status.HTTP_400_BAD_REQUEST
                )

        Microsite.objects.filter(pk=pk).update(values=request.data)

        if 's3_logo_url' in request.data:
            s3_logo_url = request.data.get('s3_logo_url', None)
            try:
                save_org_logo(s3_logo_url, course_org_filter)
                messages['logo-image-error'] = "No error."
            except Exception as e:
                messages['logo-image-error'] = '{}'.format(e)

        if serializer.is_valid():
            serializer.save()
            messages['id'] = microsite.id
            save_to_file()
            return Response(
                messages,
                status=status.HTTP_200_OK
            )
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)<|endoftext|>


<commit_before>#!/usr/bin/env python

# This script is intended to sync the performance graphs to sourceforge. It is
# essentially a large wrapper for rsync. The reason it is separated, rather
# than just have an rsync command in place is so that we make sure the SF dest
# is the same, and that we can check for errors consistently before syncing.

# rsync over ssh is used to transfer the files to SourceForge. The user running
# this script needs to have configured access to web.sourceforge.net.

import os
import posixpath
import sys
import subprocess
import shlex
import time
import argparse

def main():
    parser = argparse.ArgumentParser(description='Syncs chapel performance '
        'graphs to sourceforge. Assumes user has configured access to '
        'web.sourceforge.net. Checks for a SUCCESS file in the directory that '
        'will be synced to ensure the graphs were successfully created.')
    parser.add_argument('dirToSync', metavar='DIR', help='perf directory to '
        'sync that contains the SUCCESS file')
    parser.add_argument('destDir', metavar='DIR', help='directory on sf to '
        'sync the graphs to.')
    parser.add_argument('--logFile','-l', metavar='FILE', default=sys.stdout,
        type=argparse.FileType('w'), help='log file (default is stdout)')
    args = parser.parse_args()

    dirToSync = args.dirToSync
    destDir = args.destDir
    logFile = args.logFile

    sync = syncToSourceForge(dirToSync, destDir, logFile)
    exit(sync)

# Send the performance graphs to sourceforge
# Returns the status of the rsync command (0 on success)
# or 124 (value that doesn't conflict with rsync exit codes) if the SUCCESS
# file wasn't found in the directory that is being synced
def syncToSourceForge(dirToSync, destDir, logFile):

    logFile.write('SF sync log for: ' + time.strftime("%m/%d/%Y") + '\n\n')

    successFile = os.path.join(dirToSync, 'SUCCESS')

    if not os.path.isfile(successFile):
        logFile.write('SUCCESS file did not exist in %s. Assuming genGraph was '
          'unsuccessful. Graphs will NOT be synced.'%(dirToSync))
        logFile.close()
        return 124

    # Assumes correct username and authentication for web.sourceforge.net is
    # configured for the current system.
    sfBaseDest = 'web.sourceforge.net:/home/project-web/chapel/htdocs/perf/'
    sfPerfDest = posixpath.join(sfBaseDest, destDir)

    logFile.write('Attempting to rsync %s to %s\n'%(dirToSync, sfPerfDest))

    # The rsync command that we will execute -- authenticates over ssh
    # --del to remove any old data (graphs merged, changed names, removed etc)
    rsyncCommand = 'rsync -avz --del -e ssh ' + localPerfSrc + ' ' + sfPerfDest
    logFile.write('Sync to sourceforge command was: ' + rsyncCommand + '\n\n')
    logFile.write('Rsync output is: \n')
    logFile.flush()

    # Actually send graphs to sourceforge and pipe output to the logfile
    rsync = subprocess.call(shlex.split(rsyncCommand), stdout=logFile)

    if rsync == 0:
        logFile.write('\n\nSuccessfully sent performance graphs to sourceforge')
    else:
        logFile.write('\n\nFailure sending performance graphs to sourceforge')

    logFile.close()

    return rsync

if __name__ == "__main__":
    main()
<commit_msg>Missed changing a variable name in syncPerfGraphs<commit_after>#!/usr/bin/env python

# This script is intended to sync the performance graphs to sourceforge. It is
# essentially a large wrapper for rsync. The reason it is separated, rather
# than just have an rsync command in place is so that we make sure the SF dest
# is the same, and that we can check for errors consistently before syncing.

# rsync over ssh is used to transfer the files to SourceForge. The user running
# this script needs to have configured access to web.sourceforge.net.

import os
import posixpath
import sys
import subprocess
import shlex
import time
import argparse

def main():
    parser = argparse.ArgumentParser(description='Syncs chapel performance '
        'graphs to sourceforge. Assumes user has configured access to '
        'web.sourceforge.net. Checks for a SUCCESS file in the directory that '
        'will be synced to ensure the graphs were successfully created.')
    parser.add_argument('dirToSync', metavar='DIR', help='perf directory to '
        'sync that contains the SUCCESS file')
    parser.add_argument('destDir', metavar='DIR', help='directory on sf to '
        'sync the graphs to.')
    parser.add_argument('--logFile','-l', metavar='FILE', default=sys.stdout,
        type=argparse.FileType('w'), help='log file (default is stdout)')
    args = parser.parse_args()

    dirToSync = args.dirToSync
    destDir = args.destDir
    logFile = args.logFile

    sync = syncToSourceForge(dirToSync, destDir, logFile)
    exit(sync)

# Send the performance graphs to sourceforge
# Returns the status of the rsync command (0 on success)
# or 124 (value that doesn't conflict with rsync exit codes) if the SUCCESS
# file wasn't found in the directory that is being synced
def syncToSourceForge(dirToSync, destDir, logFile):

    logFile.write('SF sync log for: ' + time.strftime("%m/%d/%Y") + '\n\n')

    successFile = os.path.join(dirToSync, 'SUCCESS')

    if not os.path.isfile(successFile):
        logFile.write('SUCCESS file did not exist in %s. Assuming genGraph was '
          'unsuccessful. Graphs will NOT be synced.'%(dirToSync))
        logFile.close()
        return 124

    # Assumes correct username and authentication for web.sourceforge.net is
    # configured for the current system.
    sfBaseDest = 'web.sourceforge.net:/home/project-web/chapel/htdocs/perf/'
    sfPerfDest = posixpath.join(sfBaseDest, destDir)

    logFile.write('Attempting to rsync %s to %s\n'%(dirToSync, sfPerfDest))

    # The rsync command that we will execute -- authenticates over ssh
    # --del to remove any old data (graphs merged, changed names, removed etc)
    rsyncCommand = 'rsync -avz --del -e ssh ' + dirToSync + ' ' + sfPerfDest
    logFile.write('Sync to sourceforge command was: ' + rsyncCommand + '\n\n')
    logFile.write('Rsync output is: \n')
    logFile.flush()

    # Actually send graphs to sourceforge and pipe output to the logfile
    rsync = subprocess.call(shlex.split(rsyncCommand), stdout=logFile)

    if rsync == 0:
        logFile.write('\n\nSuccessfully sent performance graphs to sourceforge')
    else:
        logFile.write('\n\nFailure sending performance graphs to sourceforge')

    logFile.close()

    return rsync

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># latex2jax

import re
import sys

def extract_title(textbody):
    """
    Returns the title of the .tex file, assuming that the
    title is set in \title{TITLE}. Otherwise, returns "TITLE"
    """
    title_re = re.compile(r"^\s*\\title{([^}]*)}", re.MULTILINE)
    title = title_re.findall(textbody)
    if title:
        return title[0]
    return "TITLE"

def clean_extra_newlines(textbody):
    """Removes excess newlines"""
    multi_returns_re = re.compile(r"\n\n+")
    textbody = multi_returns_re.sub(r"\n\n", textbody)
    return textbody

def clean_whitespace(textbody):
    """Removes excess whitespaces, comments, and maketitle, toc"""
    comment_re = re.compile("%.*?$", re.MULTILINE)
    textbody = comment_re.sub("", textbody)

    begin_re = re.compile(r"\\begin\s*")
    textbody = begin_re.sub(r"\\begin", textbody)

    end_re = re.compile(r"\\end\s*")
    textbody = end_re.sub(r"\\end", textbody)

    textbody = clean_extra_newlines(textbody)

    titletoc_re = re.compile(r"\\maketitle\s*|\\tableofcontents\s*")
    textbody = titletoc_re.sub("", textbody)

    return textbody

def separate_body(textbody):
    """Separates the preamble from the document body"""
    begin_end_doc_re = re.compile(r"\\begin\{document}|\\end\{document}")
    doc = begin_end_doc_re.split(textbody)
    if len(doc) == 1:
        preamble = None
        body = doc[0]
    else:
        preamble = doc[0]
        body = doc[1]
    return preamble, body

ESC = [
    ["\\$","_dollar_","$","\\$"],
    ["\\%","_percent_","%","\\%"],
    ["\\&","_amp_","&","\\&"],
    [">","_greater_",">",">"],
    ["<","_lesser_","<","<"],
    [r"\\", "_linebreak_", "
", r"\\\\"]
]
def reformat_escapes_prelim(body):
    """Swaps \$, \%, \&, <, > for placeholders, to be handled later"""
    for e in ESC:
        body = body.replace(e[0], e[1])
    return body

def reformat_escapes_text(body):
    """Swaps the ESC placeholders in the text document."""
    for e in ESC:
        body = body.replace(e[1], e[2])
    return body

def reformat_escapes_math(math_piece):
    """Swaps the ESC placeholders in the math document."""
    for e in ESC:
        math_piece = math_piece.replace(e[1], e[3])
    return math_piece

ACCENTS = [
    #["\\\\","
\n"],
    ["\\ "," "],
    ["\\`a","à"],
    ["\\'a","á"],
    ["\\\"a","ä"],
    ["\\aa ","å"],
    ["{\\aa}","å"],
    ["\\`e","è"],
    ["\\'e","é"],
    ["\\\"e","ë"],
    ["\\`i","ì"],
    ["\\'i","í"],
    ["\\\"i","ï"],
    ["\\`o","ò"],
    ["\\'o","ó"],
    ["\\\"o","ö"],
    ["\\`o","ò"],
    ["\\'o","ó"],
    ["\\\"o","ö"],
    ["\\H o","ö"],
    ["\\`u","ù"],
    ["\\'u","ú"],
    ["\\\"u","ü"],
    ["\\`u","ù"],
    ["\\'u","ú"],
    ["\\\"u","ü"],
    ["\\v{C}","Č"]
]
def reformat_accents(body):
    """Replace TeX flavored accents with HTML compatible accents"""
    for s1, s2 in ACCENTS:
        body = body.replace(s1, s2)
    return body

fontstyle = {
  r'{\em ' : 'em',
  r'{\bf ' : 'b',
  r'{\it ' : 'i',
  r'{\sl ' : 'i',
  r'\textit{' : 'i',
  r'\textsl{' : 'i',
  r'\emph{' : 'em',
  r'\textbf{' : 'b',
}
#TODO reformat_fontstyles
def reformat_fontstyles(body):
    """Replace TeX fontstyles with HTML fontstyles"""
    pass

#TODO reformat_subsections
def reformat_sections(body):
    """Replaces sections, subsections, and subsubsections with html"""
    pass

#TODO enclose_math_environments
def separate_math(body):
    """Returns math, text split in body"""
    math_re = re.compile(r"\$+.*?\$+" +
                         r"|\\begin\{equation\*?\}.*?\\end\{equation\*?\}" +
                         r"|\\begin\{align\*?\}.*?\\end\{align\*?\}" +
                         r"|\\\[.*?\\\]",
                         flags=re.DOTALL)
    math = math_re.findall(body)
    text = math_re.split(body)
    return math, text

ThmEnvs = [
    "theorem",
    "definition",
    "lemma",
    "proposition",
    "corollary",
    "claim",
    "remark",
    "example",
    "exercise",
    "proof"
]
def handle_environments(body):
    """Handles the theorem environments in body"""
    envs_re = re.compile(r"\\begin\{\w+}" +
                         r"|\\end\{\w+}" +
                         r"|\\section\*?\s*\{.*?}" +
                         r"|\\subsection\*?\s*\{.*?}" +
                         r"|\\subsubsection\*?\s*\{.*?}" +
                         r"|\\item")
    backbody = envs_re.split(body)
    env_flags = envs_re.findall(body)
    text = backbody[0]
    i = 0
    while i < len(env_flags):
        found = False
        in_list = False

        for ThmEnv in ThmEnvs:
            if env_flags[i].find("{"+ThmEnv+"}") != -1:
                #print("Found " + ThmEnv, env_flags[i])
                found = True
                if env_flags[i].find("begin") != -1:
                    text = text + add_open_theorem_div(ThmEnv)
                elif env_flags[i].find("end") != -1:
                    text = text + "
\n\n"
                else:
                    print("Error! Type ENVFLAG")

        if env_flags[i].find("{itemize}") != -1:
            text = text + convert_itemize(env_flags[i])
            found = True
        elif env_flags[i].find("{enumerate}") != -1:
            text = text + convert_enum(env_flags[i])
            found = True
#        elif re.findall(r"^\w*\\item", env_flags[i], flags=re.MULTILINE):
#            print("found")
#            text = text + "
"
#            found = True
        elif env_flags[i][0:5] == "\\item":
            text = text + "
"
            found = True
            in_list = True
        elif env_flags[i].find("\\subsubsection") != -1:
            text = text + convert_subsubsection(env_flags[i])
            found = True
        elif env_flags[i].find("\\subsection") != -1:
            text = text + convert_subsection(env_flags[i])
            found = True
        elif env_flags[i].find("\\section") != -1:
            text = text + convert_section(env_flags[i])
            found = True
        else:
            if not found:
                print("Error! Type ENVFLAG2")
                print(env_flags[i])

        if not found:
            text = text + env_flags[i]

        if not in_list:
            text += backbody[i+1]
        else:
            text += backbody[i+1].rstrip()
            text += "
\n"

        i += 1
    return text

def add_open_theorem_div(theorem_type):
    open_div = '\n\n
'
    return open_div

def convert_itemize(m):
    """{itemize} -->
 ...
"""
    if m.find("begin") != -1 :
        return ("\n\n
")
    else :
        return ("
\n\n")

def convert_enum(m):
    """{enumerate} -->
 ...
"""
    if m.find("begin") != -1 :
        return ("\n\n
")
    else :
        return ("
\n\n")

def convert_section(m):
    braces = re.compile(r"\{|}")
    section_name = braces.split(m)[1]
    return "
" + section_name + "
"

def convert_subsection(m):
    braces = re.compile(r"\{|}")
    section_name = braces.split(m)[1]
    return "
" + section_name + "
"

def convert_subsubsection(m):
    braces = re.compile(r"\{|}")
    section_name = braces.split(m)[1]
    return "
" + section_name + "
"

def place_p_tags(body): #TODO docstring
    multi_returns_re = re.compile(r"\n\n+")
    body_pieces = multi_returns_re.split(body)
    text = ""
    for piece in body_pieces:
        if piece and piece[0] != "<":
            text += "\n\n
\n" + piece + "\n
\n\n"
        else:
            text += piece
    text = multi_returns_re.sub(r"\n\n", text)
    return text

def insert_header(body): #TODO docstring
    f = open("jax_template.html")
    base = f.read()
    f.close()
    content_re = re.compile("__CONTENT")
    text = content_re.sub(body, base)
    return text

def get_input_output_files():
    inputfile = "input.tex"
    outputfile = "output.html"
    if len(sys.argv) > 1 :
        inputfile = sys.argv[1]
        if len(sys.argv) > 2 :
            outputfile = sys.argv[2]
        else :
            if inputfile[-4:] == ".tex":
                outputfile = inputfile.replace(".tex",".html")
            elif inputfile[-3:] == ".md":
                outputfile = inputfile.replace(".md", ".html")
            else:
                outputfile = inputfile + ".html"
    return inputfile, outputfile

def driver(use_stdin_stdout=False, use_rawinput_stdout=False, rawinput=None):
    if use_stdin_stdout:
        text = sys.stdin.read()
        outputfile = None
    elif use_rawinput_stdout:
        if rawinput is None:
            raise KeyError("Rawinput is empty.")
        text = rawinput
        outputfile = None
    else:
        inputfile, outputfile = get_input_output_files()
        with open(inputfile, "r") as f:
            text = f.read()

    title = extract_title(text)

    text = reformat_escapes_prelim(text)
    text = reformat_accents(text)

    text = clean_whitespace(text)
    print(title)
    preamble, body = separate_body(text)
    print(preamble)
    print(body)

    math, body = separate_math(body)

    math = [str(m) for m in math]
    body = [str(t) for t in body]

    # Reconcatenate the text with placeholders for math.
    text = body[0]
    for i in range(len(math)):
        text = text + "__math"+str(i)+"__" + body[i+1]

    # Handle theorem environments, I suppose.
    text = handle_environments(text)
    text = reformat_escapes_text(text)
    math = [reformat_escapes_math(piece) for piece in math]
    text = place_p_tags(text)

    # TODO temporarily remove header
    #text = insert_header(text)

    text = clean_extra_newlines(text)

    # Replace math into text
    for i in range(len(math)):
        text = text.replace("__math"+str(i)+"__", math[i])

    if outputfile is None:
        print(text)
        return

    with open(outputfile, "w") as out:
        #out.write(title + "\n\n" + preamble + "\n\n" + body)
        #out.write(title + "\n\n" + preamble + "\n\n")
    #    out.write(math)
    #    out.write(body)
        out.write(text)
    print("The output is now in " + str(outputfile))
    return

if __name__=="__main__":
    driver()

"""
currently hanging:
    1. title, preamble, body currently unused in driver()
    2. handle `\\\\`, as in ACCENTS, but I removed
"""
<commit_msg>handle basic fontstyles<commit_after># latex2jax

import re
import sys

def extract_title(textbody):
    """
    Returns the title of the .tex file, assuming that the
    title is set in \title{TITLE}. Otherwise, returns "TITLE"
    """
    title_re = re.compile(r"^\s*\\title{([^}]*)}", re.MULTILINE)
    title = title_re.findall(textbody)
    if title:
        return title[0]
    return "TITLE"

def clean_extra_newlines(textbody):
    """Removes excess newlines"""
    multi_returns_re = re.compile(r"\n\n+")
    textbody = multi_returns_re.sub(r"\n\n", textbody)
    return textbody

def clean_whitespace(textbody):
    """Removes excess whitespaces, comments, and maketitle, toc"""
    comment_re = re.compile("%.*?$", re.MULTILINE)
    textbody = comment_re.sub("", textbody)

    begin_re = re.compile(r"\\begin\s*")
    textbody = begin_re.sub(r"\\begin", textbody)

    end_re = re.compile(r"\\end\s*")
    textbody = end_re.sub(r"\\end", textbody)

    textbody = clean_extra_newlines(textbody)

    titletoc_re = re.compile(r"\\maketitle\s*|\\tableofcontents\s*")
    textbody = titletoc_re.sub("", textbody)

    return textbody

def separate_body(textbody):
    """Separates the preamble from the document body"""
    begin_end_doc_re = re.compile(r"\\begin\{document}|\\end\{document}")
    doc = begin_end_doc_re.split(textbody)
    if len(doc) == 1:
        preamble = None
        body = doc[0]
    else:
        preamble = doc[0]
        body = doc[1]
    return preamble, body

ESC = [
    ["\\$","_dollar_","$","\\$"],
    ["\\%","_percent_","%","\\%"],
    ["\\&","_amp_","&","\\&"],
    [">","_greater_",">",">"],
    ["<","_lesser_","<","<"],
    [r"\\", "_linebreak_", "
", r"\\\\"]
]
def reformat_escapes_prelim(body):
    """Swaps \$, \%, \&, <, > for placeholders, to be handled later"""
    for e in ESC:
        body = body.replace(e[0], e[1])
    return body

def reformat_escapes_text(body):
    """Swaps the ESC placeholders in the text document."""
    for e in ESC:
        body = body.replace(e[1], e[2])
    return body

def reformat_escapes_math(math_piece):
    """Swaps the ESC placeholders in the math document."""
    for e in ESC:
        math_piece = math_piece.replace(e[1], e[3])
    return math_piece

ACCENTS = [
    #["\\\\","
\n"],
    ["\\ "," "],
    ["\\`a","à"],
    ["\\'a","á"],
    ["\\\"a","ä"],
    ["\\aa ","å"],
    ["{\\aa}","å"],
    ["\\`e","è"],
    ["\\'e","é"],
    ["\\\"e","ë"],
    ["\\`i","ì"],
    ["\\'i","í"],
    ["\\\"i","ï"],
    ["\\`o","ò"],
    ["\\'o","ó"],
    ["\\\"o","ö"],
    ["\\`o","ò"],
    ["\\'o","ó"],
    ["\\\"o","ö"],
    ["\\H o","ö"],
    ["\\`u","ù"],
    ["\\'u","ú"],
    ["\\\"u","ü"],
    ["\\`u","ù"],
    ["\\'u","ú"],
    ["\\\"u","ü"],
    ["\\v{C}","Č"]
]
def reformat_accents(body):
    """Replace TeX flavored accents with HTML compatible accents"""
    for s1, s2 in ACCENTS:
        body = body.replace(s1, s2)
    return body

FONTSTYLES = {
#  r'{\em ' : 'em',
#  r'{\bf ' : 'b',
#  r'{\it ' : 'i',
#  r'{\sl ' : 'i',
  r'\\textit{' : 'i',
#  r'\textsl{' : 'i',
  r'\\emph{' : 'em',
  r'\\textbf{' : 'strong',
}
def reformat_fontstyles(body):
    """Replace TeX fontstyles with HTML fontstyles"""
    meat_re = re.compile(r"{([^}]*)}")
    for key, val in FONTSTYLES.items():
        regex = key + r"[^}]*}"
        emph_re = re.compile(regex)
        matches = emph_re.findall(body)
        nonmatches = emph_re.split(body)
        ret = []
        for match in matches:
            ret.append(nonmatches.pop(0))
            matchstring = "<{}>{}</{}>".format(val, meat_re.findall(match)[0], val)
            ret.append(matchstring)
        ret.append(nonmatches[0])
        body = ''.join(ret)
    return body

#TODO reformat_subsections
def reformat_sections(body):
    """Replaces sections, subsections, and subsubsections with html"""
    pass

#TODO enclose_math_environments
def separate_math(body):
    """Returns math, text split in body"""
    math_re = re.compile(r"\$+.*?\$+" +
                         r"|\\begin\{equation\*?\}.*?\\end\{equation\*?\}" +
                         r"|\\begin\{align\*?\}.*?\\end\{align\*?\}" +
                         r"|\\\[.*?\\\]",
                         flags=re.DOTALL)
    math = math_re.findall(body)
    text = math_re.split(body)
    return math, text

ThmEnvs = [
    "theorem",
    "definition",
    "lemma",
    "proposition",
    "corollary",
    "claim",
    "remark",
    "example",
    "exercise",
    "proof"
]
def handle_environments(body):
    """Handles the theorem environments in body"""
    envs_re = re.compile(r"\\begin\{\w+}" +
                         r"|\\end\{\w+}" +
                         r"|\\section\*?\s*\{.*?}" +
                         r"|\\subsection\*?\s*\{.*?}" +
                         r"|\\subsubsection\*?\s*\{.*?}" +
                         r"|\\item")
    backbody = envs_re.split(body)
    env_flags = envs_re.findall(body)
    text = backbody[0]
    i = 0
    while i < len(env_flags):
        found = False
        in_list = False

        for ThmEnv in ThmEnvs:
            if env_flags[i].find("{"+ThmEnv+"}") != -1:
                #print("Found " + ThmEnv, env_flags[i])
                found = True
                if env_flags[i].find("begin") != -1:
                    text = text + add_open_theorem_div(ThmEnv)
                elif env_flags[i].find("end") != -1:
                    text = text + "
\n\n"
                else:
                    print("Error! Type ENVFLAG")

        if env_flags[i].find("{itemize}") != -1:
            text = text + convert_itemize(env_flags[i])
            found = True
        elif env_flags[i].find("{enumerate}") != -1:
            text = text + convert_enum(env_flags[i])
            found = True
#        elif re.findall(r"^\w*\\item", env_flags[i], flags=re.MULTILINE):
#            print("found")
#            text = text + "
"
#            found = True
        elif env_flags[i][0:5] == "\\item":
            text = text + "
"
            found = True
            in_list = True
        elif env_flags[i].find("\\subsubsection") != -1:
            text = text + convert_subsubsection(env_flags[i])
            found = True
        elif env_flags[i].find("\\subsection") != -1:
            text = text + convert_subsection(env_flags[i])
            found = True
        elif env_flags[i].find("\\section") != -1:
            text = text + convert_section(env_flags[i])
            found = True
        else:
            if not found:
                print("Error! Type ENVFLAG2")
                print(env_flags[i])

        if not found:
            text = text + env_flags[i]

        if not in_list:
            text += backbody[i+1]
        else:
            text += backbody[i+1].rstrip()
            text += "
\n"

        i += 1
    return text

def add_open_theorem_div(theorem_type):
    open_div = '\n\n
'
    return open_div

def convert_itemize(m):
    """{itemize} -->
 ...
"""
    if m.find("begin") != -1 :
        return ("\n\n
")
    else :
        return ("
\n\n")

def convert_enum(m):
    """{enumerate} -->
 ...
"""
    if m.find("begin") != -1 :
        return ("\n\n
")
    else :
        return ("
\n\n")

def convert_section(m):
    braces = re.compile(r"\{|}")
    section_name = braces.split(m)[1]
    return "
" + section_name + "
"

def convert_subsection(m):
    braces = re.compile(r"\{|}")
    section_name = braces.split(m)[1]
    return "
" + section_name + "
"

def convert_subsubsection(m):
    braces = re.compile(r"\{|}")
    section_name = braces.split(m)[1]
    return "
" + section_name + "
"

def place_p_tags(body): #TODO docstring
    multi_returns_re = re.compile(r"\n\n+")
    body_pieces = multi_returns_re.split(body)
    text = ""
    for piece in body_pieces:
        if piece and piece[0] != "<":
            text += "\n\n
\n" + piece + "\n
\n\n"
        else:
            text += piece
    text = multi_returns_re.sub(r"\n\n", text)
    return text

def insert_header(body): #TODO docstring
    f = open("jax_template.html")
    base = f.read()
    f.close()
    content_re = re.compile("__CONTENT")
    text = content_re.sub(body, base)
    return text

def get_input_output_files():
    inputfile = "input.tex"
    outputfile = "output.html"
    if len(sys.argv) > 1 :
        inputfile = sys.argv[1]
        if len(sys.argv) > 2 :
            outputfile = sys.argv[2]
        else :
            if inputfile[-4:] == ".tex":
                outputfile = inputfile.replace(".tex",".html")
            elif inputfile[-3:] == ".md":
                outputfile = inputfile.replace(".md", ".html")
            else:
                outputfile = inputfile + ".html"
    return inputfile, outputfile

def driver(use_stdin_stdout=False, use_rawinput_stdout=False, rawinput=None):
    if use_stdin_stdout:
        text = sys.stdin.read()
        outputfile = None
    elif use_rawinput_stdout:
        if rawinput is None:
            raise KeyError("Rawinput is empty.")
        text = rawinput
        outputfile = None
    else:
        inputfile, outputfile = get_input_output_files()
        with open(inputfile, "r") as f:
            text = f.read()

    title = extract_title(text)

    text = reformat_escapes_prelim(text)
    text = reformat_accents(text)

    text = clean_whitespace(text)
    #print(title)
    preamble, body = separate_body(text)
    #print(preamble)
    #print(body)

    math, body = separate_math(body)

    math = [str(m) for m in math]
    body = [str(t) for t in body]

    # Reconcatenate the text with placeholders for math.
    text = body[0]
    for i in range(len(math)):
        text = text + "__math"+str(i)+"__" + body[i+1]

    text = handle_environments(text)
    text = reformat_escapes_text(text)
    text = reformat_fontstyles(text)
    math = [reformat_escapes_math(piece) for piece in math]
    text = place_p_tags(text)

    # TODO temporarily remove header
    #text = insert_header(text)

    text = clean_extra_newlines(text)

    # Replace math into text
    for i in range(len(math)):
        text = text.replace("__math"+str(i)+"__", math[i])

    if outputfile is None:
        print(text)
        return

    with open(outputfile, "w") as out:
        #out.write(title + "\n\n" + preamble + "\n\n" + body)
        #out.write(title + "\n\n" + preamble + "\n\n")
    #    out.write(math)
    #    out.write(body)
        out.write(text)
    print("The output is now in " + str(outputfile))
    return

if __name__=="__main__":
    driver()

"""
currently hanging:
    1. title, preamble, body currently unused in driver()
    2. handle `\\\\`, as in ACCENTS, but I removed
"""
<|endoftext|>


<commit_before>import sys
import json
from powerline.colorscheme import cterm_to_hex
from itertools import groupby

if len(sys.argv) == 1:
	sys.stderr.write('''
	Usage: generate_gradients.py colors itemnum[ "show" [ min max num]]

		colors: JSON list with either cterm ([200, 42, 6]) or RGB (["abcdef",
			"feffef"]) colors.

		itemnum: number of items in generated gradient.

		"show": static string, determines whether gradient sample should be
			printed to stdout as well.

		min, max: floating point values.
		num: integer

				all of the above are used to generate sample gradient for given
				range (just like the above gradients shown with "show", but with
				different scale (controlled by min and max) and, possibly,
				different length (controlled by num)).
	''')

def linear_gradient(start_value, stop_value, start_offset, stop_offset, offset):
	return start_value + ((offset - start_offset) * (stop_value - start_value) / (stop_offset - start_offset))

def gradient(DATA):
	def gradient_function(y):
		initial_offset = 0
		for offset, start, end in DATA:
			if y <= offset:
				return [linear_gradient(start[i], end[i], initial_offset, offset, y) for i in range(3)]
			initial_offset = offset
	return gradient_function

def get_color(rgb):
	if type(rgb) is unicode:
		return int(rgb[:2], 16), int(rgb[2:4], 16), int(rgb[4:6], 16)
	else:
		return rgbint_to_rgb(cterm_to_hex[rgb])

def get_rgb(*args):
	return "%02x%02x%02x" % args

def col_distance(rgb1, rgb2):
	return sum(((rgb1[i] - rgb2[i]) ** 2 for i in range(3)))

def rgbint_to_rgb(rgbint):
	return ((rgbint >> 16) & 0xFF, (rgbint >> 8) & 0xFF, rgbint & 0xFF)

def find_color(urgb, colors):
	cur_distance = 3 * (255 ** 2 + 1)
	cur_color = None
	i = 0
	for crgbint in colors:
		crgb = rgbint_to_rgb(crgbint)
		dist = col_distance(urgb, crgb)
		if dist < cur_distance:
			cur_distance = dist
			cur_color = (i, crgb)
		i += 1
	return cur_color

def print_color(color):
	if type(color) is int:
		colstr = '5;' + str(color)
	else:
		colstr = '2;' + ';'.join((str(i) for i in color))
	sys.stdout.write('\033[48;' + colstr + 'm ')

def print_colors(colors, num=100):
	for i in range(num + 1):
		color = colors[int(round(i * (len(colors) - 1) / num))]
		print_color(color)
	sys.stdout.write('\033[0m\n')

c = [get_color(color) for color in json.loads(sys.argv[1])]
m = int(sys.argv[2]) if len(sys.argv) > 2 else 100
m += m % (len(c) - 1)
step = m / (len(c) - 1)
data = [(i * step, c[i - 1], c[i]) for i in range(1, len(c))]
gr_func = gradient(data)
gradient = [gr_func(y) for y in range(0, m - 1)]
r = [get_rgb(*color) for color in gradient]
r2 = [find_color(color, cterm_to_hex)[0] for color in gradient]
r3 = [i[0] for i in groupby(r2)]
print(json.dumps(r))
print(json.dumps(r2))
print(json.dumps(r3))
if len(sys.argv) > 3 and sys.argv[3] == 'show':
	print_colors(gradient)
	print_colors(r2)
	print_colors(r3)
	sys.stdout.write('0')
	sys.stdout.write(''.join(('%10u' % (i * 10) for i in range(1, 11))))
	sys.stdout.write('\n')
	nums = (''.join((str(i) for i in range(10))))
	sys.stdout.write(''.join(((('\033[1m' if j % 2 else '\033[0m') + nums) for j in range(10))))
	sys.stdout.write('\033[0m0\n')
	if len(sys.argv) > 6:
		vmin = float(sys.argv[4])
		vmax = float(sys.argv[5])
		num = int(sys.argv[6])
		print_colors(gradient, num)
		s = ''
		while len(s) < num:
			curpc = len(s) + 1 if s else 0
			curval = vmin + curpc * (vmax - vmin) / 100.0
			s += str(curval) + ' '
		print(s)
<commit_msg>Made tools/generate_gradients.py work in python3<commit_after>import sys
import json
from powerline.colorscheme import cterm_to_hex
from itertools import groupby

try:
	from __builtin__ import unicode
except ImportError:
	unicode = str

if len(sys.argv) == 1:
	sys.stderr.write('''
	Usage: generate_gradients.py colors itemnum[ "show" [ min max num]]

		colors: JSON list with either cterm ([200, 42, 6]) or RGB (["abcdef",
			"feffef"]) colors.

		itemnum: number of items in generated gradient.

		"show": static string, determines whether gradient sample should be
			printed to stdout as well.

		min, max: floating point values.
		num: integer

				all of the above are used to generate sample gradient for given
				range (just like the above gradients shown with "show", but with
				different scale (controlled by min and max) and, possibly,
				different length (controlled by num)).
	''')

def linear_gradient(start_value, stop_value, start_offset, stop_offset, offset):
	return start_value + ((offset - start_offset) * (stop_value - start_value) / (stop_offset - start_offset))

def gradient(DATA):
	def gradient_function(y):
		initial_offset = 0
		for offset, start, end in DATA:
			if y <= offset:
				return [linear_gradient(start[i], end[i], initial_offset, offset, y) for i in range(3)]
			initial_offset = offset
	return gradient_function

def get_color(rgb):
	if type(rgb) is unicode:
		return int(rgb[:2], 16), int(rgb[2:4], 16), int(rgb[4:6], 16)
	else:
		return rgbint_to_rgb(cterm_to_hex[rgb])

def get_rgb(*args):
	return "%02x%02x%02x" % args

def col_distance(rgb1, rgb2):
	return sum(((rgb1[i] - rgb2[i]) ** 2 for i in range(3)))

def rgbint_to_rgb(rgbint):
	return ((rgbint >> 16) & 0xFF, (rgbint >> 8) & 0xFF, rgbint & 0xFF)

def find_color(urgb, colors):
	cur_distance = 3 * (255 ** 2 + 1)
	cur_color = None
	i = 0
	for crgbint in colors:
		crgb = rgbint_to_rgb(crgbint)
		dist = col_distance(urgb, crgb)
		if dist < cur_distance:
			cur_distance = dist
			cur_color = (i, crgb)
		i += 1
	return cur_color

def print_color(color):
	if type(color) is int:
		colstr = '5;' + str(color)
	else:
		colstr = '2;' + ';'.join((str(int(round(i))) for i in color))
	sys.stdout.write('\033[48;' + colstr + 'm ')

def print_colors(colors, num=100):
	for i in range(num + 1):
		color = colors[int(round(i * (len(colors) - 1) / num))]
		print_color(color)
	sys.stdout.write('\033[0m\n')

c = [get_color(color) for color in json.loads(sys.argv[1])]
m = int(sys.argv[2]) if len(sys.argv) > 2 else 100
m += m % (len(c) - 1)
step = m / (len(c) - 1)
data = [(i * step, c[i - 1], c[i]) for i in range(1, len(c))]
gr_func = gradient(data)
gradient = [gr_func(y) for y in range(0, m - 1)]
r = [get_rgb(*color) for color in gradient]
r2 = [find_color(color, cterm_to_hex)[0] for color in gradient]
r3 = [i[0] for i in groupby(r2)]
print(json.dumps(r))
print(json.dumps(r2))
print(json.dumps(r3))
if len(sys.argv) > 3 and sys.argv[3] == 'show':
	print_colors(gradient)
	print_colors(r2)
	print_colors(r3)
	sys.stdout.write('0')
	sys.stdout.write(''.join(('%10u' % (i * 10) for i in range(1, 11))))
	sys.stdout.write('\n')
	nums = (''.join((str(i) for i in range(10))))
	sys.stdout.write(''.join(((('\033[1m' if j % 2 else '\033[0m') + nums) for j in range(10))))
	sys.stdout.write('\033[0m0\n')
	if len(sys.argv) > 6:
		vmin = float(sys.argv[4])
		vmax = float(sys.argv[5])
		num = int(sys.argv[6])
		print_colors(gradient, num)
		s = ''
		while len(s) < num:
			curpc = len(s) + 1 if s else 0
			curval = vmin + curpc * (vmax - vmin) / 100.0
			s += str(curval) + ' '
		print(s)
<|endoftext|>


<commit_before>import sys
import tty
import termios
import serial
import RPi.GPIO as GPIO
import os
import Adafruit_CharLCD as LCD
import subprocess
from subprocess import *
from time import sleep, strftime
from datetime import datetime
from engine import getch, find, rotary, update_lcd_until_enter, start_modem

# Raspberry Pi pin configuration:
lcd_rs        = 27
lcd_en        = 22
lcd_d4        = 25
lcd_d5        = 24
lcd_d6        = 23
lcd_d7        = 18
lcd_backlight = 4
lcd_columns   = 16
lcd_rows      = 2

# GPIO, serial, modem and LCD setup
port = serial.Serial("/dev/ttyAMA0", baudrate=9600, timeout=1)
lcd = LCD.Adafruit_CharLCD(lcd_rs, lcd_en, lcd_d4, lcd_d5, lcd_d6, lcd_d7, lcd_columns, lcd_rows, lcd_backlight)
GPIO.setup(21, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)  # Rotary
GPIO.setup(26, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)  # Hook
GPIO.setup(20, GPIO.OUT)  # Ringer
GPIO.output(20, GPIO.LOW)
start_modem()

def hook_lifted():
    return (GPIO.input(26) != 0)

def call():
    False

def init_call(number):
    False

def recv_call():
    None

def ringer(time):
    GPIO.output(20, GPIO.HIGH)
    sleep(time)
    GPIO.output(20, GPIO.LOW)

def gui_call():
    print("call")
    (number, status) = update_lcd_until_enter(lcd, "Enter number:\n{0}")
    if not status:
        return
    sleep(0.5)
    lcd.clear()
    sleep(0.5)
    lcd.message("Preparing...")
    status = init_call()
    if not status:
        return
    status = call()
    if not status:
        return

# --- --- --- --- --- --- --- --- --- --- --- ---

# Main loop
def main():
    ringer(1)
    while (True):
        gui_call()

if __name__ == "__main__":
    main()
<commit_msg>Lift hook to call<commit_after>import sys
import tty
import termios
import serial
import RPi.GPIO as GPIO
import os
import Adafruit_CharLCD as LCD
import subprocess
from subprocess import *
from time import sleep, strftime
from datetime import datetime
from engine import getch, find, rotary, update_lcd_until_enter, start_modem

# Raspberry Pi pin configuration:
lcd_rs        = 27
lcd_en        = 22
lcd_d4        = 25
lcd_d5        = 24
lcd_d6        = 23
lcd_d7        = 18
lcd_backlight = 4
lcd_columns   = 16
lcd_rows      = 2

# GPIO, serial, modem and LCD setup
port = serial.Serial("/dev/ttyAMA0", baudrate=9600, timeout=1)
lcd = LCD.Adafruit_CharLCD(lcd_rs, lcd_en, lcd_d4, lcd_d5, lcd_d6, lcd_d7, lcd_columns, lcd_rows, lcd_backlight)
GPIO.setup(21, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)  # Rotary
GPIO.setup(26, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)  # Hook
GPIO.setup(20, GPIO.OUT)  # Ringer
GPIO.output(20, GPIO.LOW)
start_modem()

def hook_lifted():
    return (GPIO.input(26) != 0)

def call():
    False

def init_call(number):
    False

def recv_call():
    None

def ringer(time):
    GPIO.output(20, GPIO.HIGH)
    sleep(time)
    GPIO.output(20, GPIO.LOW)

def gui_call():
    print("call")
    (number, status) = update_lcd_until_enter(lcd, "Enter number:\n{0}")
    if not status:
        return
    sleep(0.5)
    lcd.clear()
    sleep(0.5)
    lcd.message("Preparing...")
    status = init_call()
    if not status:
        return
    status = call()
    if not status:
        return

# --- --- --- --- --- --- --- --- --- --- --- ---

# Main loop
def main():
    ringer(1)
    while (True):
        sleep(0.25)
        if hook_lifted():
            gui_call()
        else:
            lcd.clear()
            lcd.message("Ready")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import errno
import os
import sys
import time

OS_ENVIRON_WHITELIST = [
    "DISPLAY",
    "EDITOR",
    "LANG",
    "LD_LIBRARY_PATH",
    "SHELL",
    "SSH_AGENT",
    "SSH_AUTH_SOCK",
    "TERM",
]

class Stop(Exception):
    """Raise to stop loops started with `loop`."""

def find_apply(funs, *args, **kw):
    for f in funs:
        result = f(*args)
        if result is not None:
            return result
    return kw.get("default")

def ensure_dir(d):
    try:
        os.makedirs(d)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

def pid_exists(pid):
    try:
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True

def free_port():
    import random
    import socket
    min_port = 49152
    max_port = 65535
    max_attempts = 100
    attempts = 0

    while True:
        if attempts > max_attempts:
            raise RuntimeError("too many free port attempts")
        port = random.randint(min_port, max_port)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(0.1)
        try:
            sock.connect(('localhost', port))
        except socket.timeout:
            return port
        except socket.error as e:
            if e.errno == errno.ECONNREFUSED:
                return port
        else:
            sock.close()
        attempts += 1

def open_url(url):
    try:
        _open_url_with_cmd(url)
    except OSError:
        _open_url_with_webbrowser(url)

def _open_url_with_cmd(url):
    import subprocess
    if sys.platform == "darwin":
        args = ["open", url]
    else:
        args = ["xdg-open", url]
    with open(os.devnull, "w") as null:
        subprocess.check_call(args, stderr=null, stdout=null)

def _open_url_with_webbrowser(url):
    import webbrowser
    webbrowser.open(url)

def loop(cb, wait, interval, first_interval=None):
    try:
        _loop(cb, wait, interval, first_interval)
    except Stop:
        pass
    except KeyboardInterrupt:
        pass

def _loop(cb, wait, interval, first_interval):
    loop_interval = first_interval if first_interval is not None else interval
    start = time.time()
    while True:
        sleep = _sleep_interval(loop_interval, start)
        loop_interval = interval
        should_stop = wait(sleep)
        if should_stop:
            break
        cb()

def _sleep_interval(interval, start):
    if interval <= 0:
        return 0
    now_ms = int(time.time() * 1000)
    interval_ms = int(interval * 1000)
    start_ms = int(start * 1000)
    sleep_ms = (
        ((now_ms - start_ms) // interval_ms + 1)
        * interval_ms + start_ms - now_ms)
    return sleep_ms / 1000

def safe_osenv():
    return {
        name: val
        for name, val in os.environ.items()
        if name in OS_ENVIRON_WHITELIST
    }
<commit_msg>Let env pass through<commit_after># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import errno
import os
import sys
import time

OS_ENVIRON_BLACKLIST = []

class Stop(Exception):
    """Raise to stop loops started with `loop`."""

def find_apply(funs, *args, **kw):
    for f in funs:
        result = f(*args)
        if result is not None:
            return result
    return kw.get("default")

def ensure_dir(d):
    try:
        os.makedirs(d)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

def pid_exists(pid):
    try:
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True

def free_port():
    import random
    import socket
    min_port = 49152
    max_port = 65535
    max_attempts = 100
    attempts = 0

    while True:
        if attempts > max_attempts:
            raise RuntimeError("too many free port attempts")
        port = random.randint(min_port, max_port)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(0.1)
        try:
            sock.connect(('localhost', port))
        except socket.timeout:
            return port
        except socket.error as e:
            if e.errno == errno.ECONNREFUSED:
                return port
        else:
            sock.close()
        attempts += 1

def open_url(url):
    try:
        _open_url_with_cmd(url)
    except OSError:
        _open_url_with_webbrowser(url)

def _open_url_with_cmd(url):
    import subprocess
    if sys.platform == "darwin":
        args = ["open", url]
    else:
        args = ["xdg-open", url]
    with open(os.devnull, "w") as null:
        subprocess.check_call(args, stderr=null, stdout=null)

def _open_url_with_webbrowser(url):
    import webbrowser
    webbrowser.open(url)

def loop(cb, wait, interval, first_interval=None):
    try:
        _loop(cb, wait, interval, first_interval)
    except Stop:
        pass
    except KeyboardInterrupt:
        pass

def _loop(cb, wait, interval, first_interval):
    loop_interval = first_interval if first_interval is not None else interval
    start = time.time()
    while True:
        sleep = _sleep_interval(loop_interval, start)
        loop_interval = interval
        should_stop = wait(sleep)
        if should_stop:
            break
        cb()

def _sleep_interval(interval, start):
    if interval <= 0:
        return 0
    now_ms = int(time.time() * 1000)
    interval_ms = int(interval * 1000)
    start_ms = int(start * 1000)
    sleep_ms = (
        ((now_ms - start_ms) // interval_ms + 1)
        * interval_ms + start_ms - now_ms)
    return sleep_ms / 1000

def safe_osenv():
    return {
        name: val
        for name, val in os.environ.items()
        if name not in OS_ENVIRON_BLACKLIST
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import requests
import json

_REQUEST_TIMEOUT_SECS = 10

def _make_pod_config(pod_name, image_name, container_port_list, cmd_list,
                    arg_list):
  """Creates a string containing the Pod defintion as required by the Kubernetes API"""
  body = {
      'kind': 'Pod',
      'apiVersion': 'v1',
      'metadata': {
          'name': pod_name,
          'labels': {'name': pod_name}
      },
      'spec': {
          'containers': [
              {
                  'name': pod_name,
                  'image': image_name,
                  'ports': []
              }
          ]
      }
  }
  # Populate the 'ports' list
  for port in container_port_list:
    port_entry = {'containerPort': port, 'protocol': 'TCP'}
    body['spec']['containers'][0]['ports'].append(port_entry)

  # Add the 'Command' and 'Args' attributes if they are passed.
  # Note:
  #  - 'Command' overrides the ENTRYPOINT in the Docker Image
  #  - 'Args' override the COMMAND in Docker image (yes, it is confusing!)
  if len(cmd_list) > 0:
    body['spec']['containers'][0]['command'] = cmd_list
  if len(arg_list) > 0:
    body['spec']['containers'][0]['args'] = arg_list
  return json.dumps(body)

def _make_service_config(service_name, pod_name, service_port_list,
                        container_port_list, is_headless):
  """Creates a string containing the Service definition as required by the Kubernetes API.

  NOTE:
  This creates either a Headless Service or 'LoadBalancer' service depending on
  the is_headless parameter. For Headless services, there is no 'type' attribute
  and the 'clusterIP' attribute is set to 'None'. Also, if the service is
  Headless, Kubernetes creates DNS entries for Pods - i.e creates DNS A-records
  mapping the service's name to the Pods' IPs
  """
  if len(container_port_list) != len(service_port_list):
    print(
        'ERROR: container_port_list and service_port_list must be of same size')
    return ''
  body = {
      'kind': 'Service',
      'apiVersion': 'v1',
      'metadata': {
          'name': service_name,
          'labels': {
              'name': service_name
          }
      },
      'spec': {
          'ports': [],
          'selector': {
              'name': pod_name
          }
      }
  }
  # Populate the 'ports' list in the 'spec' section. This maps service ports
  # (port numbers that are exposed by Kubernetes) to container ports (i.e port
  # numbers that are exposed by your Docker image)
  for idx in range(len(container_port_list)):
    port_entry = {
        'port': service_port_list[idx],
        'targetPort': container_port_list[idx],
        'protocol': 'TCP'
    }
    body['spec']['ports'].append(port_entry)

  # Make this either a LoadBalancer service or a headless service depending on
  # the is_headless parameter
  if is_headless:
    body['spec']['clusterIP'] = 'None'
  else:
    body['spec']['type'] = 'LoadBalancer'
  return json.dumps(body)

def _print_connection_error(msg):
  print('ERROR: Connection failed. Did you remember to run Kubenetes proxy on'
        'localhost (i.e kubectl proxy --port=<proxy_port>) ?. Error: %s' % msg)

def _do_post(post_url, api_name, request_body):
  """Helper to do HTTP POST.

  Note:
  1) On success, Kubernetes returns a success code of 201(CREATED) not 200(OK)
  2) A response code of 509(CONFLICT) is interpreted as a success code (since
  the error is most likely due to the resource already existing). This makes
  _do_post() idempotent which is semantically desirable.
  """
  is_success = True
  try:
    r = requests.post(post_url, data=request_body, timeout=_REQUEST_TIMEOUT_SECS)
    if r.status_code == requests.codes.conflict:
      print('WARN: Looks like the resource already exists. Api: %s, url: %s' %
            (api_name, post_url))
    elif r.status_code != requests.codes.created:
      print('ERROR: %s API returned error. HTTP response: (%d) %s' %
            (api_name, r.status_code, r.text))
      is_success = False
  except(requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
    is_success = False
    _print_connection_error(str(e))
  return is_success

def _do_delete(del_url, api_name):
  """Helper to do HTTP DELETE.

  Note: A response code of 404(NOT_FOUND) is treated as success to keep
  _do_delete() idempotent.
  """
  is_success = True
  try:
    r = requests.delete(del_url, timeout=_REQUEST_TIMEOUT_SECS)
    if r.status_code == requests.codes.not_found:
      print('WARN: The resource does not exist. Api: %s, url: %s' %
            (api_name, del_url))
    elif r.status_code != requests.codes.ok:
      print('ERROR: %s API returned error. HTTP response: %s' %
            (api_name, r.text))
      is_success = False
  except(requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
    is_success = False
    _print_connection_error(str(e))
  return is_success

def create_service(kube_host, kube_port, namespace, service_name, pod_name,
                   service_port_list, container_port_list, is_headless):
  """Creates either a Headless Service or a LoadBalancer Service depending
  on the is_headless parameter.
  """
  post_url = 'http://%s:%d/api/v1/namespaces/%s/services' % (
      kube_host, kube_port, namespace)
  request_body = _make_service_config(service_name, pod_name, service_port_list,
                                     container_port_list, is_headless)
  return _do_post(post_url, 'Create Service', request_body)

def create_pod(kube_host, kube_port, namespace, pod_name, image_name,
               container_port_list, cmd_list, arg_list):
  """Creates a Kubernetes Pod.

  Note that it is generally NOT considered a good practice to directly create
  Pods. Typically, the recommendation is to create 'Controllers' to create and
  manage Pods' lifecycle. Currently Kubernetes only supports 'Replication
  Controller' which creates a configurable number of 'identical Replicas' of
  Pods and automatically restarts any Pods in case of failures (for eg: Machine
  failures in Kubernetes). This makes it less flexible for our test use cases
  where we might want slightly different set of args to each Pod. Hence we
  directly create Pods and not care much about Kubernetes failures since those
  are very rare.
  """
  post_url = 'http://%s:%d/api/v1/namespaces/%s/pods' % (kube_host, kube_port,
                                                         namespace)
  request_body = _make_pod_config(pod_name, image_name, container_port_list,
                                 cmd_list, arg_list)
  return do_post(post_url, 'Create Pod', request_body)

def delete_service(kube_host, kube_port, namespace, service_name):
  del_url = 'http://%s:%d/api/v1/namespaces/%s/services/%s' % (
      kube_host, kube_port, namespace, service_name)
  return _do_delete(del_url, 'Delete Service')

def delete_pod(kube_host, kube_port, namespace, pod_name):
  del_url = 'http://%s:%d/api/v1/namespaces/%s/pods/%s' % (kube_host, kube_port,
                                                           namespace, pod_name)
  return _do_delete(del_url, 'Delete Pod')
<commit_msg>Fix a typo<commit_after>#!/usr/bin/env python2.7
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import requests
import json

_REQUEST_TIMEOUT_SECS = 10

def _make_pod_config(pod_name, image_name, container_port_list, cmd_list,
                    arg_list):
  """Creates a string containing the Pod defintion as required by the Kubernetes API"""
  body = {
      'kind': 'Pod',
      'apiVersion': 'v1',
      'metadata': {
          'name': pod_name,
          'labels': {'name': pod_name}
      },
      'spec': {
          'containers': [
              {
                  'name': pod_name,
                  'image': image_name,
                  'ports': []
              }
          ]
      }
  }
  # Populate the 'ports' list
  for port in container_port_list:
    port_entry = {'containerPort': port, 'protocol': 'TCP'}
    body['spec']['containers'][0]['ports'].append(port_entry)

  # Add the 'Command' and 'Args' attributes if they are passed.
  # Note:
  #  - 'Command' overrides the ENTRYPOINT in the Docker Image
  #  - 'Args' override the COMMAND in Docker image (yes, it is confusing!)
  if len(cmd_list) > 0:
    body['spec']['containers'][0]['command'] = cmd_list
  if len(arg_list) > 0:
    body['spec']['containers'][0]['args'] = arg_list
  return json.dumps(body)

def _make_service_config(service_name, pod_name, service_port_list,
                        container_port_list, is_headless):
  """Creates a string containing the Service definition as required by the Kubernetes API.

  NOTE:
  This creates either a Headless Service or 'LoadBalancer' service depending on
  the is_headless parameter. For Headless services, there is no 'type' attribute
  and the 'clusterIP' attribute is set to 'None'. Also, if the service is
  Headless, Kubernetes creates DNS entries for Pods - i.e creates DNS A-records
  mapping the service's name to the Pods' IPs
  """
  if len(container_port_list) != len(service_port_list):
    print(
        'ERROR: container_port_list and service_port_list must be of same size')
    return ''
  body = {
      'kind': 'Service',
      'apiVersion': 'v1',
      'metadata': {
          'name': service_name,
          'labels': {
              'name': service_name
          }
      },
      'spec': {
          'ports': [],
          'selector': {
              'name': pod_name
          }
      }
  }
  # Populate the 'ports' list in the 'spec' section. This maps service ports
  # (port numbers that are exposed by Kubernetes) to container ports (i.e port
  # numbers that are exposed by your Docker image)
  for idx in range(len(container_port_list)):
    port_entry = {
        'port': service_port_list[idx],
        'targetPort': container_port_list[idx],
        'protocol': 'TCP'
    }
    body['spec']['ports'].append(port_entry)

  # Make this either a LoadBalancer service or a headless service depending on
  # the is_headless parameter
  if is_headless:
    body['spec']['clusterIP'] = 'None'
  else:
    body['spec']['type'] = 'LoadBalancer'
  return json.dumps(body)

def _print_connection_error(msg):
  print('ERROR: Connection failed. Did you remember to run Kubenetes proxy on'
        'localhost (i.e kubectl proxy --port=<proxy_port>) ?. Error: %s' % msg)

def _do_post(post_url, api_name, request_body):
  """Helper to do HTTP POST.

  Note:
  1) On success, Kubernetes returns a success code of 201(CREATED) not 200(OK)
  2) A response code of 509(CONFLICT) is interpreted as a success code (since
  the error is most likely due to the resource already existing). This makes
  _do_post() idempotent which is semantically desirable.
  """
  is_success = True
  try:
    r = requests.post(post_url, data=request_body, timeout=_REQUEST_TIMEOUT_SECS)
    if r.status_code == requests.codes.conflict:
      print('WARN: Looks like the resource already exists. Api: %s, url: %s' %
            (api_name, post_url))
    elif r.status_code != requests.codes.created:
      print('ERROR: %s API returned error. HTTP response: (%d) %s' %
            (api_name, r.status_code, r.text))
      is_success = False
  except(requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
    is_success = False
    _print_connection_error(str(e))
  return is_success

def _do_delete(del_url, api_name):
  """Helper to do HTTP DELETE.

  Note: A response code of 404(NOT_FOUND) is treated as success to keep
  _do_delete() idempotent.
  """
  is_success = True
  try:
    r = requests.delete(del_url, timeout=_REQUEST_TIMEOUT_SECS)
    if r.status_code == requests.codes.not_found:
      print('WARN: The resource does not exist. Api: %s, url: %s' %
            (api_name, del_url))
    elif r.status_code != requests.codes.ok:
      print('ERROR: %s API returned error. HTTP response: %s' %
            (api_name, r.text))
      is_success = False
  except(requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
    is_success = False
    _print_connection_error(str(e))
  return is_success

def create_service(kube_host, kube_port, namespace, service_name, pod_name,
                   service_port_list, container_port_list, is_headless):
  """Creates either a Headless Service or a LoadBalancer Service depending
  on the is_headless parameter.
  """
  post_url = 'http://%s:%d/api/v1/namespaces/%s/services' % (
      kube_host, kube_port, namespace)
  request_body = _make_service_config(service_name, pod_name, service_port_list,
                                     container_port_list, is_headless)
  return _do_post(post_url, 'Create Service', request_body)

def create_pod(kube_host, kube_port, namespace, pod_name, image_name,
               container_port_list, cmd_list, arg_list):
  """Creates a Kubernetes Pod.

  Note that it is generally NOT considered a good practice to directly create
  Pods. Typically, the recommendation is to create 'Controllers' to create and
  manage Pods' lifecycle. Currently Kubernetes only supports 'Replication
  Controller' which creates a configurable number of 'identical Replicas' of
  Pods and automatically restarts any Pods in case of failures (for eg: Machine
  failures in Kubernetes). This makes it less flexible for our test use cases
  where we might want slightly different set of args to each Pod. Hence we
  directly create Pods and not care much about Kubernetes failures since those
  are very rare.
  """
  post_url = 'http://%s:%d/api/v1/namespaces/%s/pods' % (kube_host, kube_port,
                                                         namespace)
  request_body = _make_pod_config(pod_name, image_name, container_port_list,
                                 cmd_list, arg_list)
  return _do_post(post_url, 'Create Pod', request_body)

def delete_service(kube_host, kube_port, namespace, service_name):
  del_url = 'http://%s:%d/api/v1/namespaces/%s/services/%s' % (
      kube_host, kube_port, namespace, service_name)
  return _do_delete(del_url, 'Delete Service')

def delete_pod(kube_host, kube_port, namespace, pod_name):
  del_url = 'http://%s:%d/api/v1/namespaces/%s/pods/%s' % (kube_host, kube_port,
                                                           namespace, pod_name)
  return _do_delete(del_url, 'Delete Pod')
<|endoftext|>


<commit_before>import sys
import os
import pathlib
import map_helpers

#main("../../_maps/")
def main(map_folder, tgm=0):
    list_of_files = list()
    for root, directories, filenames in os.walk(map_folder):
        for filename in [f for f in filenames if f.endswith(".dmm")]:
            list_of_files.append(pathlib.Path(root, filename))

    for i in range(0, len(list_of_files)):
        to_print = "[{}]: {}".format(i, list_of_files[i])
        print(to_print)
        print("".join("-" for _ in range(len(to_print))))

    in_list = input("List the maps you want to merge (example: 1,2,3,4,5):\n")
    in_list = in_list.replace(" ", "")
    in_list = in_list.split(",")

    valid_indices = list()
    for m in in_list:
        index = string_to_num(m)
        if index >= 0 and index < len(list_of_files):
            valid_indices.append(index)

    if tgm == "1":
        print("\nMaps will be converted to tgm.")
        tgm = True
    else:
        print("\nMaps will not be converted to tgm.")
        tgm = False

    print("\nMerging these maps:")
    for i in valid_indices:
        print(list_of_files[i])
    merge = input("\nPress Enter to merge...")
    if merge == "abort":
        print("\nAborted map merge.")
        sys.exit()
    else:
        for i in valid_indices:
            path_str = str(list_of_files[i])
            try:
                if map_helpers.merge_map(path_str, path_str + ".backup", tgm) != 1:
                    print("ERROR MERGING: {}".format(list_of_files[i]))
                    continue
                print("MERGED: {}".format(path_str))
            except FileNotFoundError:
                print("\nERROR: File not found! Make sure you run 'Prepare Maps.bat' before merging.")
                print(path_str + " || " + path_str+".backup")

    print("\nFinished merging.")

def string_to_num(s):
    try:
        return int(s)
    except ValueError:
        return -1

main(sys.argv[1], sys.argv[2])
<commit_msg>Made mapmerge slightly less of a pain<commit_after>import sys
import os
import pathlib
import map_helpers

#main("../../_maps/")
def main(map_folder, tgm=0):
    list_of_files = list()
    for root, directories, filenames in os.walk(map_folder):
        for filename in [f for f in filenames if f.endswith(".dmm")]:
            list_of_files.append(pathlib.Path(root, filename))

    last_dir = ""
    for i in range(0, len(list_of_files)):
        this_dir = list_of_files[i].parent
        if last_dir != this_dir:
            print("--------------------------------")
            last_dir = this_dir
        print("[{}]: {}".format(i, str(list_of_files[i])[len(map_folder):]))

    print("--------------------------------")
    in_list = input("List the maps you want to merge (example: 1,3-5,12):\n")
    in_list = in_list.replace(" ", "")
    in_list = in_list.split(",")

    valid_indices = list()
    for m in in_list:
        index_range = m.split("-")
        if len(index_range) == 1:
            index = string_to_num(index_range[0])
            if index >= 0 and index < len(list_of_files):
                valid_indices.append(index)
        elif len(index_range) == 2:
            index0 = string_to_num(index_range[0])
            index1 = string_to_num(index_range[1])
            if index0 >= 0 and index0 <= index1 and index1 < len(list_of_files):
                valid_indices.extend(range(index0, index1 + 1))

    if tgm == "1":
        print("\nMaps will be converted to tgm.")
        tgm = True
    else:
        print("\nMaps will not be converted to tgm.")
        tgm = False

    print("\nMerging these maps:")
    for i in valid_indices:
        print(str(list_of_files[i])[len(map_folder):])
    merge = input("\nPress Enter to merge...")
    if merge == "abort":
        print("\nAborted map merge.")
        sys.exit()
    else:
        for i in valid_indices:
            path_str = str(list_of_files[i])
            path_str_pretty = path_str[len(map_folder):]
            try:
                if map_helpers.merge_map(path_str, path_str + ".backup", tgm) != 1:
                    print("ERROR MERGING: {}".format(path_str_pretty))
                    continue
                print("MERGED: {}".format(path_str_pretty))
            except FileNotFoundError:
                print("\nERROR: File not found! Make sure you run 'Prepare Maps.bat' before merging.")
                print(path_str_pretty + " || " + path_str_pretty + ".backup")

    print("\nFinished merging.")

def string_to_num(s):
    try:
        return int(s)
    except ValueError:
        return -1

main(sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before>from bowser.Bot import Bot

def main():
    bot = Bot()
    try:
        token = open('token.txt').read().replace('\n', '')
        bot.run(token)
    except Exception as ex:
        bot.loop.run_until_complete(bot.close())
        raise ex

def init():
    if __name__ == '__main__':
        main()

init()
<commit_msg>test: Remove one of the unclosed client session warnings<commit_after>from bowser.Bot import Bot

def main():
    bot = Bot()
    try:
        token = open('token.txt').read().replace('\n', '')
        bot.run(token)
    except Exception as ex:
        raise ex
    finally:
        bot.loop.run_until_complete(bot.close())

def init():
    if __name__ == '__main__':
        main()

init()
<|endoftext|>