<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import celery
from celery.result import AsyncResult

from girder import events, logger
from girder.constants import AccessType
from girder.models.model_base import ValidationException
from girder.plugins.jobs.constants import JobStatus
from girder.utility import setting_utilities
from girder.utility.model_importer import ModelImporter

from .constants import PluginSettings
from .utils import getWorkerApiUrl, jobInfoSpec

_celeryapp = None

class CustomJobStatus(object):
    """
    The custom job status flags for the worker.
    """
    FETCHING_INPUT = 820
    CONVERTING_INPUT = 821
    CONVERTING_OUTPUT = 822
    PUSHING_OUTPUT = 823
    CANCELING = 824

    valid_transitions = {
        JobStatus.QUEUED: [JobStatus.INACTIVE],
        JobStatus.RUNNING: [JobStatus.QUEUED],
        FETCHING_INPUT: [JobStatus.RUNNING],
        CONVERTING_INPUT: [JobStatus.RUNNING, FETCHING_INPUT],
        CONVERTING_OUTPUT: [JobStatus.RUNNING],
        PUSHING_OUTPUT: [JobStatus.RUNNING, CONVERTING_OUTPUT],
        CANCELING: [JobStatus.INACTIVE, JobStatus.QUEUED, JobStatus.RUNNING],
        JobStatus.ERROR: [FETCHING_INPUT, CONVERTING_INPUT, CONVERTING_OUTPUT,
                          PUSHING_OUTPUT, CANCELING, JobStatus.QUEUED,
                          JobStatus.RUNNING],
        # The last two are allowed for revoke called from outside Girder
        JobStatus.CANCELED: [CANCELING, JobStatus.QUEUED, JobStatus.RUNNING],
        JobStatus.SUCCESS: [JobStatus.RUNNING, PUSHING_OUTPUT]
    }

    @classmethod
    def isValid(cls, status):
        return status in (
            cls.FETCHING_INPUT,
            cls.CONVERTING_INPUT,
            cls.CONVERTING_OUTPUT,
            cls.PUSHING_OUTPUT,
            cls.CANCELING
        )

    @classmethod
    def validTransitions(cls, status):
        return cls.valid_transitions.get(status)

def getCeleryApp():
    """
    Lazy loader for the celery app. Reloads anytime the settings are updated.
    """
    global _celeryapp

    if _celeryapp is None:
        settings = ModelImporter.model('setting')
        backend = settings.get(PluginSettings.BACKEND) or 'amqp://guest@localhost/'
        broker = settings.get(PluginSettings.BROKER) or 'amqp://guest@localhost/'
        _celeryapp = celery.Celery('girder_worker', backend=backend, broker=broker)
    return _celeryapp

def schedule(event):
    """
    This is bound to the "jobs.schedule" event, and will be triggered any time
    a job is scheduled. This handler will process any job that has the
    handler field set to "worker_handler".
    """
    job = event.info
    if job['handler'] == 'worker_handler':
        # Stop event propagation since we have taken care of scheduling.
        event.stopPropagation()

        task = job.get('celeryTaskName', 'girder_worker.run')

        # Set the job status to queued
        ModelImporter.model('job', 'jobs').updateJob(job, status=JobStatus.QUEUED)

        # Send the task to celery
        asyncResult = getCeleryApp().send_task(
            task, job['args'], job['kwargs'], queue=job.get('celeryQueue'), headers={
                'jobInfoSpec': jobInfoSpec(job, job.get('token', None)),
                'apiUrl': getWorkerApiUrl()
            })

        # Record the task ID from celery.
        ModelImporter.model('job', 'jobs').updateJob(job, otherFields={
            'celeryTaskId': asyncResult.task_id
        })

def cancel(event):
    """
    This is bound to the "jobs.cancel" event, and will be triggered any time
    a job is canceled. This handler will process any job that has the
    handler field set to "worker_handler".
    """
    job = event.info
    if job['handler'] == 'worker_handler':
        # Stop event propagation and prevent default, we are using a custom state
        event.stopPropagation().preventDefault()

        celeryTaskId = job.get('celeryTaskId')

        if celeryTaskId is None:
            msg = ("Unable to cancel Celery task. Job '%s' doesn't have a Celery task id."
                   % job['_id'])
            logger.warn(msg)
            return

        if job['status'] not in [JobStatus.SUCCESS, JobStatus.ERROR]:
            # Set the job status to canceling
            ModelImporter.model('job', 'jobs').updateJob(job, status=CustomJobStatus.CANCELING)

            # Send the revoke request.
            asyncResult = AsyncResult(celeryTaskId)
            asyncResult.revoke()

@setting_utilities.validator({
    PluginSettings.BROKER,
    PluginSettings.BACKEND
})
def validateSettings(doc):
    """
    Handle plugin-specific system settings. Right now we don't do any
    validation for the broker or backend URL settings, but we do reinitialize
    the celery app object with the new values.
    """
    global _celeryapp
    _celeryapp = None

@setting_utilities.validator({
    PluginSettings.API_URL
})
def validateApiUrl(doc):
    val = doc['value']
    if val and not val.startswith('http://') and not val.startswith('https://'):
        raise ValidationException('API URL must start with http:// or https://.', 'value')

def validateJobStatus(event):
    """Allow our custom job status values."""
    if CustomJobStatus.isValid(event.info):
        event.preventDefault().addResponse(True)

def validTransitions(event):
    """Allow our custom job transitions."""
    if event.info['job']['handler'] == 'worker_handler':
            states = CustomJobStatus.validTransitions(event.info['status'])
            event.preventDefault().addResponse(states)

def load(info):
    events.bind('jobs.schedule', 'worker', schedule)
    events.bind('jobs.status.validate', 'worker', validateJobStatus)
    events.bind('jobs.status.validTransitions', 'worker', validTransitions)
    events.bind('jobs.cancel', 'worker', cancel)

    ModelImporter.model('job', 'jobs').exposeFields(
        AccessType.SITE_ADMIN, {'celeryTaskId', 'celeryQueue'})
<commit_msg>Add state transitions for celery_handler<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import celery
from celery.result import AsyncResult

from girder import events, logger
from girder.constants import AccessType
from girder.models.model_base import ValidationException
from girder.plugins.jobs.constants import JobStatus
from girder.utility import setting_utilities
from girder.utility.model_importer import ModelImporter

from .constants import PluginSettings
from .utils import getWorkerApiUrl, jobInfoSpec

_celeryapp = None

class CustomJobStatus(object):
    """
    The custom job status flags for the worker.
    """
    FETCHING_INPUT = 820
    CONVERTING_INPUT = 821
    CONVERTING_OUTPUT = 822
    PUSHING_OUTPUT = 823
    CANCELING = 824

    # valid transitions for worker scheduled jobs
    valid_worker_transitions = {
        JobStatus.QUEUED: [JobStatus.INACTIVE],
        JobStatus.RUNNING: [JobStatus.QUEUED],
        FETCHING_INPUT: [JobStatus.RUNNING],
        CONVERTING_INPUT: [JobStatus.RUNNING, FETCHING_INPUT],
        CONVERTING_OUTPUT: [JobStatus.RUNNING],
        PUSHING_OUTPUT: [JobStatus.RUNNING, CONVERTING_OUTPUT],
        CANCELING: [JobStatus.INACTIVE, JobStatus.QUEUED, JobStatus.RUNNING],
        JobStatus.ERROR: [FETCHING_INPUT, CONVERTING_INPUT, CONVERTING_OUTPUT,
                          PUSHING_OUTPUT, CANCELING, JobStatus.QUEUED,
                          JobStatus.RUNNING],
        # The last two are allowed for revoke called from outside Girder
        JobStatus.CANCELED: [CANCELING, JobStatus.QUEUED, JobStatus.RUNNING],
        JobStatus.SUCCESS: [JobStatus.RUNNING, PUSHING_OUTPUT]
    }

    # valid transitions for celery scheduled jobs
    valid_celery_transitions = {
        JobStatus.QUEUED: [JobStatus.INACTIVE],
        CANCELING: [JobStatus.INACTIVE, JobStatus.QUEUED],
        JobStatus.CANCELED: [CANCELING, JobStatus.QUEUED, JobStatus.RUNNING],
        JobStatus.RUNNING: [JobStatus.INACTIVE, JobStatus.QUEUED],
        JobStatus.ERROR: [JobStatus.QUEUED, JobStatus.RUNNING],
        JobStatus.SUCCESS: [JobStatus.RUNNING]
    }

    @classmethod
    def isValid(cls, status):
        return status in (
            cls.FETCHING_INPUT,
            cls.CONVERTING_INPUT,
            cls.CONVERTING_OUTPUT,
            cls.PUSHING_OUTPUT,
            cls.CANCELING
        )

    @classmethod
    def validTransitionsWorker(cls, status):
        return cls.valid_worker_transitions.get(status)

    @classmethod
    def validTransitionsCelery(cls, status):
        return cls.valid_celery_transitions.get(status)

def getCeleryApp():
    """
    Lazy loader for the celery app. Reloads anytime the settings are updated.
    """
    global _celeryapp

    if _celeryapp is None:
        settings = ModelImporter.model('setting')
        backend = settings.get(PluginSettings.BACKEND) or 'amqp://guest@localhost/'
        broker = settings.get(PluginSettings.BROKER) or 'amqp://guest@localhost/'
        _celeryapp = celery.Celery('girder_worker', backend=backend, broker=broker)
    return _celeryapp

def schedule(event):
    """
    This is bound to the "jobs.schedule" event, and will be triggered any time
    a job is scheduled. This handler will process any job that has the
    handler field set to "worker_handler".
    """
    job = event.info
    if job['handler'] == 'worker_handler':
        # Stop event propagation since we have taken care of scheduling.
        event.stopPropagation()

        task = job.get('celeryTaskName', 'girder_worker.run')

        # Set the job status to queued
        ModelImporter.model('job', 'jobs').updateJob(job, status=JobStatus.QUEUED)

        # Send the task to celery
        asyncResult = getCeleryApp().send_task(
            task, job['args'], job['kwargs'], queue=job.get('celeryQueue'), headers={
                'jobInfoSpec': jobInfoSpec(job, job.get('token', None)),
                'apiUrl': getWorkerApiUrl()
            })

        # Record the task ID from celery.
        ModelImporter.model('job', 'jobs').updateJob(job, otherFields={
            'celeryTaskId': asyncResult.task_id
        })

def cancel(event):
    """
    This is bound to the "jobs.cancel" event, and will be triggered any time
    a job is canceled. This handler will process any job that has the
    handler field set to "worker_handler".
    """
    job = event.info
    if job['handler'] in ['worker_handler', 'celery_handler']:
        # Stop event propagation and prevent default, we are using a custom state
        event.stopPropagation().preventDefault()

        celeryTaskId = job.get('celeryTaskId')

        if celeryTaskId is None:
            msg = ("Unable to cancel Celery task. Job '%s' doesn't have a Celery task id."
                   % job['_id'])
            logger.warn(msg)
            return

        if job['status'] not in [JobStatus.SUCCESS, JobStatus.ERROR]:
            # Set the job status to canceling
            ModelImporter.model('job', 'jobs').updateJob(job, status=CustomJobStatus.CANCELING)

            # Send the revoke request.
            asyncResult = AsyncResult(celeryTaskId)
            asyncResult.revoke()

@setting_utilities.validator({
    PluginSettings.BROKER,
    PluginSettings.BACKEND
})
def validateSettings(doc):
    """
    Handle plugin-specific system settings. Right now we don't do any
    validation for the broker or backend URL settings, but we do reinitialize
    the celery app object with the new values.
    """
    global _celeryapp
    _celeryapp = None

@setting_utilities.validator({
    PluginSettings.API_URL
})
def validateApiUrl(doc):
    val = doc['value']
    if val and not val.startswith('http://') and not val.startswith('https://'):
        raise ValidationException('API URL must start with http:// or https://.', 'value')

def validateJobStatus(event):
    """Allow our custom job status values."""
    if CustomJobStatus.isValid(event.info):
        event.preventDefault().addResponse(True)

def validTransitions(event):
    """Allow our custom job transitions."""
    if event.info['job']['handler'] == 'worker_handler':
        states = CustomJobStatus.validTransitionsWorker(event.info['status'])
    elif event.info['job']['handler'] == 'celery_handler':
        states = CustomJobStatus.validTransitionsCelery(event.info['status'])

    event.preventDefault().addResponse(states)

def load(info):
    events.bind('jobs.schedule', 'worker', schedule)
    events.bind('jobs.status.validate', 'worker', validateJobStatus)
    events.bind('jobs.status.validTransitions', 'worker', validTransitions)
    events.bind('jobs.cancel', 'worker', cancel)

    ModelImporter.model('job', 'jobs').exposeFields(
        AccessType.SITE_ADMIN, {'celeryTaskId', 'celeryQueue'})
<|endoftext|>


<commit_before>"""Maya initialisation for Mindbender pipeline"""

from maya import cmds

def setup():
    assert __import__("pyblish_maya").is_setup(), (
        "mindbender-core depends on pyblish_maya which has not "
        "yet been setup. Run pyblish_maya.setup()")

    from pyblish import api
    api.register_gui("pyblish_lite")

    from mindbender import api, maya
    api.install(maya)

# Allow time for dependencies (e.g. pyblish-maya)
# to be installed first.
cmds.evalDeferred(setup)
<commit_msg>update userSetup     add fps set     add format of width and height variable     add time set of animation<commit_after>"""Maya initialisation for Mindbender pipeline"""

from maya import cmds

import os
import mayafpsconverter

MINDBENDER_FPS = mayafpsconverter(os.getenv("MINDBENDER_FPS"))
MINDBENDER_EDIT_IN = os.getenv("MINDBENDER_EDIT_IN")
MINDBENDER_EDIT_OUT = os.getenv("MINDBENDER_EDIT_OUT")
MINDBENDER_RESOLUTION_WIDTH = os.getenv("MINDBENDER_RESOLUTION_WIDTH")
MINDBENDER_RESOLUTION_HEIGHT = os.getenv("MINDBENDER_RESOLUTION_HEIGHT")

def setup():
    assert __import__("pyblish_maya").is_setup(), (
        "mindbender-core depends on pyblish_maya which has not "
        "yet been setup. Run pyblish_maya.setup()")

    from pyblish import api
    api.register_gui("pyblish_lite")

    from mindbender import api, maya
    api.install(maya)

# Allow time for dependencies (e.g. pyblish-maya)
# to be installed first.
cmds.evalDeferred(setup)
if not MINDBENDER_FPS == "":
    cmds.currentUnit(time=MINDBENDER_FPS)
if not MINDBENDER_EDIT_IN == "":
    cmds.playbackOption(animationEndTime=MINDBENDER_EDIT_IN)
if not MINDBENDER_EDIT_OUT == "":
    cmds.playbackOption(animationEndTime=MINDBENDER_EDIT_OUT)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# anywhere.py ---
#
# Filename: anywhere.py
# Description:
# Author: Werther Zhang
# Maintainer:
# Created: Mon Jul 24 13:37:15 2017 (+0800)
#

# Change Log:
#
#

import sys
import os
import urllib
import urllib2
import cookielib
import requests
import json
from six.moves import _thread
import time

SHARED_DIR_ROOT='/extdisk/sda1/'
DOWNLOAD_DIR='/extdisks/sda1/下载/'
#DOWNLOAD_DIR='/home/werther/Music/'

class SendAnywhere (object):
    API_ENDPOINTS = "https://send-anywhere.com"
    RESPONSE_CODE = {
        200:"OK",
        400:"Bad Request. Invalid request query string.",
        403:"Bad Request. Invalid request query string.",
        404:"Not Found. Invalid key",
        429:"Too Many Request. To keep the amount of spam on Send Anywhere as low as possible.",
        500:"Internal Server Error. Something went wrong on our side. We're very sorry."
    }

    def __init__(self, api_key, msg=None):
        self.cookie = cookielib.CookieJar()
        self.handler = urllib2.HTTPCookieProcessor(self.cookie)
        self.opener = urllib2.build_opener(self.handler)
        self.api_key = api_key
        self.download_progress = 1.0
        self.msg = msg
        self.inited = False

    def __update(self):
        request_url = "https://send-anywhere.com/web/v1/device?api_key={}&profile_name=miroute".format(self.api_key)
        response = self.opener.open(request_url)
        self.inited = True

    def __upload_file(self, url, filename, timeout):
        try:
            with open(filename, 'rb') as f:
                response = requests.post(url, files={'file': f}, timeout=timeout)
        except requests.exceptions.Timeout:
            print "Timeout waiting for post"
        print("Send file done response {}".format(response))

    def __upload_file_stream(self, url, filename, timeout):
        import poster
        poster.streaminghttp.register_openers()
        with open(filename, 'rb') as f:
            datagen, headers = poster.encode.multipart_encode({'file': f}, timeout=timeout)
            request = urllib2.Request(url, datagen, headers)
            resp = None
            try:
                resp = urllib2.urlopen(request)
            except urllib2.HTTPError as error:
                print(error)
                print(error.fp.read())

        print("Send file done response {}".format(resp))

    def send(self, filename):
        if not self.inited:
            self.__update()

        key_api_url = "{}/web/v1/key".format(SendAnywhere.API_ENDPOINTS)

        response = self.opener.open(key_api_url)
        jdata = json.loads(response.read())
        print("share key: {}".format(jdata['key']))
        #print("{}".format(jdata))
        timeout = jdata['expires_time'] - jdata['created_time']
        _thread.start_new_thread(self.__upload_file_stream,
                                 (jdata['weblink'], filename, timeout))
        #self.__upload_file(jdata['weblink'], filename, timeout)
        #time.sleep(timeout)
        return (timeout, jdata['key'])

    def __download_callback(self, downsize, totalsize):
        self.download_progress = float(downsize) / float(totalsize)

    def __download_file(self, url, fpath, filesize, callback=None):
        count = 0
        blocksize = 1024*64
        r = requests.get(url, stream=True)
        with open(fpath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=blocksize):
                if chunk:
                    f.write(chunk)
                    f.flush()
                    if callback:
                        count += 1
                        callback(count*blocksize, filesize)
        callback(filesize, filesize)
        u = url + '&mode=status&_={}'.format(int(time.time()))
        #print('status url {}'.format(u))
        res = self.opener.open(u)
        jdata = json.loads(res.read())
        dirname = os.path.dirname(fpath)
        fp = jdata.get('name', fpath).encode('utf-8')
        #print('jdata {}'.format(jdata))
        if fp != fpath:
            fp = os.path.join(dirname, fp)
            os.rename(fpath, fp)
        if self.msg:
            self.msg.reply("Finish downloading {} !".format(fp))

    def receive(self, name, key, msg=None):
        if 1 - self.download_progress > 0.01:
            print("Download in progress, please wait!!")
            return self.download_progress
        self.msg = msg
        key_api_url = "{}/web/v1/key/{}".format(SendAnywhere.API_ENDPOINTS, key)
        response = self.opener.open(key_api_url)
        jdata = json.loads(response.read())
        #print("{}".format(jdata))
        _thread.start_new_thread(self.__download_file,
                                 (jdata['weblink'],
                                  name,
                                  jdata['file_size'],
                                  self.__download_callback))
        return 0

if __name__ == '__main__':
    s = SendAnywhere("")
    #timeout, key = s.send('Lonely.mp3')
    # print("shared key : {}".format(key))
    # time.sleep(timeout)
    s.receive('mp3', 156724)
    sys.exit(1)

import zipfile
def unzip(zippath, dest):
    zippath = os.path.join(DOWNLOAD_DIR, zippath)
    fullunzipdirname = os.path.join(DOWNLOAD_DIR, dest)
    zfile = zipfile.ZipFile(zippath, 'r')
    for eachfile in zfile.namelist():
        eachfilename = os.path.normpath(os.path.join(fullunzipdirname, eachfile))
        eachdirname = os.path.dirname(eachfilename)
        if eachfile.endswith("/"):
            # dir
            os.makedirs(eachfilename)
            continue
        if not os.path.exists(eachdirname):
            os.makedirs(eachdirname)
        fd = open(eachfilename, 'wb')
        fd.write(zfile.read(eachfile))
        fd.close()

from slackbot.bot import plugin_init
from slackbot.bot import respond_to

_enable = False
_anywhere = None
@plugin_init
def init_anywhere(config):
    global _enable
    global _anywhere
    api_key = config.get('api', '')
    _enable = config.get('enable', False)
    _anywhere = SendAnywhere(api_key)

@respond_to(r'anywhere (.*)')
def anywhere_command(message, rest):
    global _enable
    global _anywhere
    if not _enable:
        message.reply("SendAnywhere not enabled")
        return

    argv = ['anywhere'] + rest.split()
    print("{}".format(argv))
    command = argv[1]
    if command == 'send':
        # anywhere send [filepath]
        if len(argv) < 3:
            message.reply("Usage: anywhere send  [filepath]")
        if argv[2] == None:
            message.reply("Usage: anywhere send  [filepath]")
        filepath = ' '.join(argv[2:])
        (timeout, key) = _anywhere.send(os.path.join(DOWNLOAD_DIR, filepath))
        message.reply('Shared file key {} expired in {} seconds'.format(key, timeout))
        return

    if command == 'receive':
        # anywhere recv key filename
        if len(argv) < 4:
            message.reply("Usage: anywhere receive [key] [filename]")
        if argv[2] == None or argv[3] == None:
            message.reply("Usage: anywhere receive [key] [filename]")
        key = argv[2]
        filename = ' '.join(argv[3:])
        filename = os.path.join(DOWNLOAD_DIR, key + '_' + filename)
        ret = _anywhere.receive(filename, key, message)
        if ret == 0:
            message.reply("Start downloading {} to {}".format(key, filename))
        else:
            message.reply("Last download not done, progress {}".format(ret))
        return

    if command == 'ls':
        print("command ls")
        if len(argv) == 2:
            argv = argv + ['{}'.format(DOWNLOAD_DIR)]
        for arg in argv[2:] :
            files = os.listdir(arg)
            message.reply('{}'.format('\n'.join(files[:])))
        return

    if command == 'unzip':
        print("command unzip")
        if len(argv) != 3 and len(argv) != 4:
            message.reply("Usage: anywhere unzip [zipfile]")
            return
        zippath = argv[2]
        if len(argv) == 4:
            dest = argv[4]
        else:
            dest = os.path.dirname(zippath)
        try:
            unzip(zippath, dest)
        except Exception as e:
            print("{}".format(e))
            message.reply("Unzip Failed")
            t, v, tb = sys.exc_info()
            raise t, v, tb
            return
        message.reply("Unzip success")
        return

    message.reply("Usage: The ROOT is {}\n"
                  "     anywhere ls\n"
                  "     anywhere send [filepath]\n"
                  "     anywhere receive [key] [filename]\n"
                  "     anywhere unzip [zipfile]".format(DOWNLOAD_DIR))
<commit_msg>update anywhere<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# anywhere.py ---
#
# Filename: anywhere.py
# Description:
# Author: Werther Zhang
# Maintainer:
# Created: Mon Jul 24 13:37:15 2017 (+0800)
#

# Change Log:
#
#

import sys
import os
import urllib
import urllib2
import cookielib
import requests
import json
from six.moves import _thread
import time

SHARED_DIR_ROOT='/extdisk/sda1/'
DOWNLOAD_DIR='/extdisks/sda1/下载/'
#DOWNLOAD_DIR='/home/werther/Music/'

class SendAnywhere (object):
    API_ENDPOINTS = "https://send-anywhere.com"
    RESPONSE_CODE = {
        200:"OK",
        400:"Bad Request. Invalid request query string.",
        403:"Bad Request. Invalid request query string.",
        404:"Not Found. Invalid key",
        429:"Too Many Request. To keep the amount of spam on Send Anywhere as low as possible.",
        500:"Internal Server Error. Something went wrong on our side. We're very sorry."
    }

    def __init__(self, api_key, msg=None):
        self.cookie = cookielib.CookieJar()
        self.handler = urllib2.HTTPCookieProcessor(self.cookie)
        self.opener = urllib2.build_opener(self.handler)
        self.api_key = api_key
        self.download_progress = 1.0
        self.msg = msg
        self.inited = False

    def __update(self):
        request_url = "https://send-anywhere.com/web/v1/device?api_key={}&profile_name=miroute".format(self.api_key)
        response = self.opener.open(request_url)
        self.inited = True

    def __upload_file(self, url, filename, timeout):
        try:
            with open(filename, 'rb') as f:
                response = requests.post(url, files={'file': f}, timeout=timeout)
        except requests.exceptions.Timeout:
            print "Timeout waiting for post"
        print("Send file done response {}".format(response))

    def __upload_file_stream(self, url, filename, timeout):
        import poster
        poster.streaminghttp.register_openers()
        with open(filename, 'rb') as f:
            datagen, headers = poster.encode.multipart_encode({'file': f})
            request = urllib2.Request(url, datagen, headers)
            resp = None
            try:
                resp = urllib2.urlopen(request, timeout=timeout)
            except urllib2.HTTPError as error:
                print(error)
                print(error.fp.read())

        print("Send file done response {}".format(resp))

    def send(self, filename):
        if not self.inited:
            self.__update()

        key_api_url = "{}/web/v1/key".format(SendAnywhere.API_ENDPOINTS)

        response = self.opener.open(key_api_url)
        jdata = json.loads(response.read())
        print("share key: {}".format(jdata['key']))
        #print("{}".format(jdata))
        timeout = jdata['expires_time'] - jdata['created_time']
        _thread.start_new_thread(self.__upload_file_stream,
                                 (jdata['weblink'], filename, timeout))
        #self.__upload_file(jdata['weblink'], filename, timeout)
        #time.sleep(timeout)
        return (timeout, jdata['key'])

    def __download_callback(self, downsize, totalsize):
        self.download_progress = float(downsize) / float(totalsize)

    def __download_file(self, url, fpath, filesize, callback=None):
        count = 0
        blocksize = 1024*64
        r = requests.get(url, stream=True)
        with open(fpath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=blocksize):
                if chunk:
                    f.write(chunk)
                    f.flush()
                    if callback:
                        count += 1
                        callback(count*blocksize, filesize)
        callback(filesize, filesize)
        u = url + '&mode=status&_={}'.format(int(time.time()))
        #print('status url {}'.format(u))
        res = self.opener.open(u)
        jdata = json.loads(res.read())
        dirname = os.path.dirname(fpath)
        fp = jdata.get('name', fpath).encode('utf-8')
        #print('jdata {}'.format(jdata))
        if fp != fpath:
            fp = os.path.join(dirname, fp)
            os.rename(fpath, fp)
        if self.msg:
            self.msg.reply("Finish downloading {} !".format(fp))

    def receive(self, name, key, msg=None):
        if 1 - self.download_progress > 0.01:
            print("Download in progress, please wait!!")
            return self.download_progress
        self.msg = msg
        key_api_url = "{}/web/v1/key/{}".format(SendAnywhere.API_ENDPOINTS, key)
        response = self.opener.open(key_api_url)
        jdata = json.loads(response.read())
        #print("{}".format(jdata))
        _thread.start_new_thread(self.__download_file,
                                 (jdata['weblink'],
                                  name,
                                  jdata['file_size'],
                                  self.__download_callback))
        return 0

if __name__ == '__main__':
    s = SendAnywhere("")
    #timeout, key = s.send('Lonely.mp3')
    # print("shared key : {}".format(key))
    # time.sleep(timeout)
    s.receive('mp3', 156724)
    sys.exit(1)

import zipfile
def unzip(zippath, dest):
    zippath = os.path.join(DOWNLOAD_DIR, zippath)
    fullunzipdirname = os.path.join(DOWNLOAD_DIR, dest)
    zfile = zipfile.ZipFile(zippath, 'r')
    for eachfile in zfile.namelist():
        eachfilename = os.path.normpath(os.path.join(fullunzipdirname, eachfile))
        eachdirname = os.path.dirname(eachfilename)
        if eachfile.endswith("/"):
            # dir
            os.makedirs(eachfilename)
            continue
        if not os.path.exists(eachdirname):
            os.makedirs(eachdirname)
        fd = open(eachfilename, 'wb')
        fd.write(zfile.read(eachfile))
        fd.close()

from slackbot.bot import plugin_init
from slackbot.bot import respond_to

_enable = False
_anywhere = None
@plugin_init
def init_anywhere(config):
    global _enable
    global _anywhere
    api_key = config.get('api', '')
    _enable = config.get('enable', False)
    _anywhere = SendAnywhere(api_key)

@respond_to(r'anywhere (.*)')
def anywhere_command(message, rest):
    global _enable
    global _anywhere
    if not _enable:
        message.reply("SendAnywhere not enabled")
        return

    argv = ['anywhere'] + rest.split()
    print("{}".format(argv))
    command = argv[1]
    if command == 'send':
        # anywhere send [filepath]
        if len(argv) < 3:
            message.reply("Usage: anywhere send  [filepath]")
        if argv[2] == None:
            message.reply("Usage: anywhere send  [filepath]")
        filepath = ' '.join(argv[2:])
        (timeout, key) = _anywhere.send(os.path.join(DOWNLOAD_DIR, filepath))
        message.reply('Shared file key {} expired in {} seconds'.format(key, timeout))
        return

    if command == 'receive':
        # anywhere recv key filename
        if len(argv) < 4:
            message.reply("Usage: anywhere receive [key] [filename]")
        if argv[2] == None or argv[3] == None:
            message.reply("Usage: anywhere receive [key] [filename]")
        key = argv[2]
        filename = ' '.join(argv[3:])
        filename = os.path.join(DOWNLOAD_DIR, key + '_' + filename)
        ret = _anywhere.receive(filename, key, message)
        if ret == 0:
            message.reply("Start downloading {} to {}".format(key, filename))
        else:
            message.reply("Last download not done, progress {}".format(ret))
        return

    if command == 'ls':
        print("command ls")
        if len(argv) == 2:
            argv = argv + ['{}'.format(DOWNLOAD_DIR)]
        for arg in argv[2:] :
            files = os.listdir(arg)
            message.reply('{}'.format('\n'.join(files[:])))
        return

    if command == 'unzip':
        print("command unzip")
        if len(argv) != 3 and len(argv) != 4:
            message.reply("Usage: anywhere unzip [zipfile]")
            return
        zippath = argv[2]
        if len(argv) == 4:
            dest = argv[4]
        else:
            dest = os.path.dirname(zippath)
        try:
            unzip(zippath, dest)
        except Exception as e:
            print("{}".format(e))
            message.reply("Unzip Failed")
            t, v, tb = sys.exc_info()
            raise t, v, tb
            return
        message.reply("Unzip success")
        return

    message.reply("Usage: The ROOT is {}\n"
                  "     anywhere ls\n"
                  "     anywhere send [filepath]\n"
                  "     anywhere receive [key] [filename]\n"
                  "     anywhere unzip [zipfile]".format(DOWNLOAD_DIR))
<|endoftext|>


<commit_before>import re
from datetime import datetime
import dateutil
import logging

log = logging.getLogger(__name__)

def dot_split(s):
    return [part for part in re.split(r"(?<!\.)\.(?!\.)", s)]

def split_strip(_str, on=',', remove_empty=True):
    lst = (_str if isinstance(_str, list) else _str.split(on))
    lst = [e.strip() for e in lst]
    if remove_empty:
        lst = filter(bool, lst)
    return lst

def str2rdt(strdt):
    matches = dict(
        s = 'seconds',
        m = 'minutes',
        h = 'hours',
        d = 'days',
        M = 'months',
        y = 'years'
    )

    # is it a relative date ?
    rg = re.compile('(([-+]?)(\d+))([smhdMy])\\b', re.DOTALL)
    m = rg.search(strdt)
    if m:
        number = int(m.group(1))
        word = m.group(4)
        if word in matches:
            return dateutil.relativedelta.relativedelta(**{matches[word]:number})

def str2dt(strdt, _raise=False):
    if not strdt:
        raise ValueError('Datetime string can not be empty or None')

    if isinstance(strdt, datetime):
        return datetime

    dt = str2rdt(strdt)
    if dt:
        return datetime.utcnow()+dt

    try:
        return dateutil.parser.parse(strdt)
    except ValueError as e:
        msg = 'Datetime string `%s` not recognized as datetime. Did you miss +- signs for relative dates?' % strdt
        if _raise:
            raise ValueError(msg)
        else:
            log.error(msg)<commit_msg>* fixed a bug in str2dt<commit_after>import re
from datetime import datetime
import dateutil
import logging

log = logging.getLogger(__name__)

def dot_split(s):
    return [part for part in re.split(r"(?<!\.)\.(?!\.)", s)]

def split_strip(_str, on=',', remove_empty=True):
    lst = (_str if isinstance(_str, list) else _str.split(on))
    lst = [e.strip() for e in lst]
    if remove_empty:
        lst = filter(bool, lst)
    return lst

def str2rdt(strdt):
    matches = dict(
        s = 'seconds',
        m = 'minutes',
        h = 'hours',
        d = 'days',
        M = 'months',
        y = 'years'
    )

    # is it a relative date ?
    rg = re.compile('(([-+]?)(\d+))([smhdMy])\\b', re.DOTALL)
    m = rg.search(strdt)
    if m:
        number = int(m.group(1))
        word = m.group(4)
        if word in matches:
            return dateutil.relativedelta.relativedelta(**{matches[word]:number})

def str2dt(strdt, _raise=False):
    if not strdt:
        raise ValueError('Datetime string can not be empty or None')

    if isinstance(strdt, datetime):
        return strdt

    dt = str2rdt(strdt)
    if dt:
        return datetime.utcnow()+dt

    try:
        return dateutil.parser.parse(strdt)
    except ValueError as e:
        msg = 'Datetime string `%s` not recognized as datetime. Did you miss +- signs for relative dates?' % strdt
        if _raise:
            raise ValueError(msg)
        else:
            log.error(msg)<|endoftext|>


<commit_before>from __future__ import absolute_import
import hashlib
import json
import re
import socket as _socket
import six.moves.urllib_parse as urlparse
import traceback
import chardet

import slyd.splash.utils

from twisted.python import log
from jsonschema.exceptions import ValidationError

from splash.browser_tab import JsError

from slyd.utils.projects import ProjectModifier
from .utils import open_tab, extract_data, BaseWSError, BadRequest, NotFound

_VIEWPORT_RE = re.compile('^\d{3,5}x\d{3,5}$')

@open_tab
def load_page(data, socket):
    """Load page in virtual url from provided url"""
    if 'url' not in data:
        return {'error': 4001, 'message': 'Required parameter url'}

    socket.tab.loaded = False

    def on_complete(error):
        extra_meta = {'id': data.get('_meta', {}).get('id')}
        if error:
            extra_meta.update(error=4500, message='Unknown error')
        else:
            socket.tab.loaded = True
        socket.sendMessage(metadata(socket, extra_meta))

    socket.tab.go(data['url'],
                  lambda: on_complete(False),
                  lambda: on_complete(True),
                  baseurl=data.get('baseurl', data['url']))

@open_tab
def interact_page(data, socket):
    """Execute JS event from front end on virtual tab"""
    event = json.dumps(data.get('interaction', {}))
    try:
        socket.tab.evaljs('window.livePortiaPage.sendEvent(%s);' % event)
    except JsError as e:
        print(e)

def resolve(data, socket):
    result = {'id': data.get('_meta', {}).get('id')}
    try:
        url = data['url']
        parsed = urlparse.urlparse(url)
        port = 443 if parsed.scheme == 'https' else 80
        _socket.getaddrinfo(parsed.hostname, port)
    except KeyError:
        result['error'] = 'Can\'t create a spider without a start url'
    except _socket.gaierror:
        result['error'] = 'Could not resolve "%s"' % url
    return result

def metadata(socket, extra={}):
    socket.tab.loaded = True
    res = {
        '_command': 'metadata',
        'loaded': socket.tab.loaded
    }
    if socket.tab.loaded:
        # Workarround for https://github.com/scrapinghub/splash/issues/259
        url = socket.tab.evaljs('location.href')
        res.update(
            url=url,
            fp=hashlib.sha1(url).hexdigest(),
            response={
                'headers': {},  # TODO: Get headers
                'status': socket.tab.last_http_status()
            }
        )
        if socket.spiderspec:
            res.update(extract(socket))
    res.update(extra)
    return res

def extract(socket):
    """Run spider on page URL to get extracted links and items"""
    if socket.tab is None or not socket.tab.loaded:
        return {
            'items': [],
            'links': {},
        }
    templates = socket.spiderspec.templates
    url = str(socket.tab.url)
    html = socket.tab.html()
    js_items, js_links = extract_data(url, html, socket.spider, templates)
    raw_html = getattr(socket.tab, '_raw_html', None)
    if raw_html:
        raw_items, links = extract_data(url, raw_html, socket.spider,
                                        templates)
    else:
        raw_items = []
        links = []
    raw = {l: 'raw' for l in links}
    js = {l: 'js' for l in js_links}
    js.update(raw)
    items = js_items
    if not (socket.spider.js_enabled and socket.spider._filter_js_urls(url)):
        items = raw_items
    return {
        'items': items,
        'links': js,
    }

def resize(data, socket):
    """Resize virtual tab viewport to match user's viewport"""
    try:
        if 'size' in data and _VIEWPORT_RE.search(data['size']):
            socket.tab.set_viewport(data['size'])
    except (KeyError, AttributeError):
        pass  # Tab isn't open. The size will be set when opened

def close_tab(data, socket):
    """Close virtual tab if it is open"""
    if socket.tab is not None:
        socket.tab.close()
        socket.factory[socket].tab = None

class ProjectData(ProjectModifier):
    errors = slyd.splash.utils

    def save_spider(self, data, socket):
        spider, meta = data.get('spider'), data.get('_meta')
        return self.save_data(['spiders', meta.get('spider')], 'spider',
                              data=spider, socket=socket, meta=meta)

    def save_template(self, data, socket):
        sample, meta = data.get('template'), data.get('_meta')
        path = ['spiders', meta.get('spider'), sample.get('name')]
        if sample.pop('_new', False):
            if socket.spider is None:
                socket.open_spider(meta)
            if socket.spider._filter_js_urls(sample['url']):
                sample['original_body'] = socket.tab.html().decode('utf-8')
            else:
                sample['original_body'] = self._decode(socket.tab._raw_html)
        return self.save_data(path, 'template', data=sample, socket=socket,
                              meta=meta)

    def save_extractors(self, data, socket):
        extractors, meta = data.get('extractors'), data.get('_meta')
        return self.save_data(['extractors'], data=extractors, socket=socket,
                              meta=meta)

    def save_items(self, data, socket):
        items, meta = data.get('items'), data.pop('_meta', None)
        return self.save_data(['items'], data=items, socket=socket, meta=meta)

    def save_data(self, path, type=None, data=None, socket=None, meta=None):
        if type is None:
            type = path[0]
        if any(v is None for v in (data, meta, socket)):
            raise BadRequest('No data provided')
        spec = socket.spec_manager.project_spec(meta['project'],
                                                socket.user.auth)
        try:
            obj = self.verify_data(path, data, spec)
        except (KeyError, IndexError) as ex:
            raise NotFound('"%s.json" could not be found' % '/'.join(path))
        except ValidationError as ex:
            print(('Not Valid: %s' % ex))
            raise BadRequest(str(ex))
        except BaseWSError as ex:
            print(('Other: %s' % ex))
            raise ex
        except Exception as ex:
            # XXX: Catch any other errors and log them leaving Websocket open
            log.err(traceback.format_exc(ex))
        else:
            try:
                spec.savejson(obj, [s.encode('utf-8') for s in path])
            except Exception as ex:
                # XXX: Catch errors in saving to the backend
                log.err(traceback.format_exc(ex))
            socket.update_spider(meta, **{type: obj})
            return obj

    def _decode(self, html):
        for encoding in ('utf-8', 'windows-1252'):
            try:
                return html.decode(encoding)
            except UnicodeDecodeError:
                pass
        encoding = chardet.detect(html).get('encoding')
        return html.decode(encoding)

def update_project_data(data, socket):
    try:
        updater = ProjectData()
        meta = data['_meta'] or {}
        option = meta.get('type', '')
        command = getattr(updater, 'save_%s' % option)
    except KeyError:
        raise BadRequest('No metadata received')
    except AttributeError:
        raise BadRequest('Unknown option "%s" received' % option)
    else:
        resp = {'saved': {option: command(data, socket)}}
    id = meta.get('id')
    if id:
        resp['id'] = id
    resp.update(extract(socket))
    return resp

def delete_project_data(data, socket):
    """Use project and spider metadata along with a resource name to delete
    spiders and templates
    """
    try:
        meta = data['_meta']
        spec = socket.spec_manager.project_spec(meta['project'],
                                                socket.user.auth)
        option = meta.get('type', '')
        command = getattr(spec, 'remove_%s' % option)
        spider = meta['spider'].encode('utf-8')
        name = data.get('name').encode('utf-8')
    except KeyError:
        raise BadRequest('No or incorrect metadata received')
    except AttributeError:
        raise BadRequest('Unknown option "%s" received' % option)
    else:
        if option == 'spider':
            command(spider)
        else:
            command(spider, name)
            if socket.spiderspec:
                socket.open_spider(meta)
    resp = {'id': meta.get('id')}
    resp.update(extract(socket))
    return resp

def rename_project_data(data, socket):
    """Use project and spider metadata along with old and new names to rename
    spiders and templates
    """
    try:
        meta = data['_meta']
        spec = socket.spec_manager.project_spec(meta['project'],
                                                socket.user.auth)
        option = meta.get('type', '')
        command = getattr(spec, 'rename_%s' % option)
        spider = meta['spider'].encode('utf-8')
        old = data['old'].encode('utf-8')
        new = data['new'].encode('utf-8')
    except KeyError:
        raise BadRequest('No or incorrect metadata received')
    except AttributeError:
        raise BadRequest('Unknown option "%s" received' % option)
    else:
        if option == 'spider':
            command(old, new)
            if socket.spiderspec:
                socket.spiderspec.name = new
        else:
            command(spider, old, new)
            if socket.spiderspec:
                socket.open_spider(meta)
    return {'id': meta.get('id')}
<commit_msg>Use encoded stated by page as first decode attempt<commit_after>from __future__ import absolute_import
import hashlib
import json
import re
import socket as _socket
import six.moves.urllib_parse as urlparse
import traceback
import chardet
import six
import itertools

import slyd.splash.utils

from twisted.python import log
from jsonschema.exceptions import ValidationError

from splash.browser_tab import JsError

from slyd.utils.projects import ProjectModifier
from .utils import open_tab, extract_data, BaseWSError, BadRequest, NotFound

_VIEWPORT_RE = re.compile('^\d{3,5}x\d{3,5}$')

@open_tab
def load_page(data, socket):
    """Load page in virtual url from provided url"""
    if 'url' not in data:
        return {'error': 4001, 'message': 'Required parameter url'}

    socket.tab.loaded = False

    def on_complete(error):
        extra_meta = {'id': data.get('_meta', {}).get('id')}
        if error:
            extra_meta.update(error=4500, message='Unknown error')
        else:
            socket.tab.loaded = True
        socket.sendMessage(metadata(socket, extra_meta))

    socket.tab.go(data['url'],
                  lambda: on_complete(False),
                  lambda: on_complete(True),
                  baseurl=data.get('baseurl', data['url']))

@open_tab
def interact_page(data, socket):
    """Execute JS event from front end on virtual tab"""
    event = json.dumps(data.get('interaction', {}))
    try:
        socket.tab.evaljs('window.livePortiaPage.sendEvent(%s);' % event)
    except JsError as e:
        print(e)

def resolve(data, socket):
    result = {'id': data.get('_meta', {}).get('id')}
    try:
        url = data['url']
        parsed = urlparse.urlparse(url)
        port = 443 if parsed.scheme == 'https' else 80
        _socket.getaddrinfo(parsed.hostname, port)
    except KeyError:
        result['error'] = 'Can\'t create a spider without a start url'
    except _socket.gaierror:
        result['error'] = 'Could not resolve "%s"' % url
    return result

def metadata(socket, extra={}):
    socket.tab.loaded = True
    res = {
        '_command': 'metadata',
        'loaded': socket.tab.loaded
    }
    if socket.tab.loaded:
        # Workarround for https://github.com/scrapinghub/splash/issues/259
        url = socket.tab.evaljs('location.href')
        res.update(
            url=url,
            fp=hashlib.sha1(url).hexdigest(),
            response={
                'headers': {},  # TODO: Get headers
                'status': socket.tab.last_http_status()
            }
        )
        if socket.spiderspec:
            res.update(extract(socket))
    res.update(extra)
    return res

def extract(socket):
    """Run spider on page URL to get extracted links and items"""
    if socket.tab is None or not socket.tab.loaded:
        return {
            'items': [],
            'links': {},
        }
    templates = socket.spiderspec.templates
    url = str(socket.tab.url)
    html = socket.tab.html()
    js_items, js_links = extract_data(url, html, socket.spider, templates)
    raw_html = getattr(socket.tab, '_raw_html', None)
    if raw_html:
        raw_items, links = extract_data(url, raw_html, socket.spider,
                                        templates)
    else:
        raw_items = []
        links = []
    raw = {l: 'raw' for l in links}
    js = {l: 'js' for l in js_links}
    js.update(raw)
    items = js_items
    if not (socket.spider.js_enabled and socket.spider._filter_js_urls(url)):
        items = raw_items
    return {
        'items': items,
        'links': js,
    }

def resize(data, socket):
    """Resize virtual tab viewport to match user's viewport"""
    try:
        if 'size' in data and _VIEWPORT_RE.search(data['size']):
            socket.tab.set_viewport(data['size'])
    except (KeyError, AttributeError):
        pass  # Tab isn't open. The size will be set when opened

def close_tab(data, socket):
    """Close virtual tab if it is open"""
    if socket.tab is not None:
        socket.tab.close()
        socket.factory[socket].tab = None

class ProjectData(ProjectModifier):
    errors = slyd.splash.utils

    def save_spider(self, data, socket):
        spider, meta = data.get('spider'), data.get('_meta')
        return self.save_data(['spiders', meta.get('spider')], 'spider',
                              data=spider, socket=socket, meta=meta)

    def save_template(self, data, socket):
        sample, meta = data.get('template'), data.get('_meta')
        path = ['spiders', meta.get('spider'), sample.get('name')]
        if sample.pop('_new', False):
            if socket.spider is None:
                socket.open_spider(meta)
            if socket.spider._filter_js_urls(sample['url']):
                sample['original_body'] = socket.tab.html().decode('utf-8')
            else:
                stated_encoding = socket.tab.evaljs('document.characterSet')
                sample['original_body'] = self._decode(socket.tab._raw_html,
                                                       stated_encoding)
        return self.save_data(path, 'template', data=sample, socket=socket,
                              meta=meta)

    def save_extractors(self, data, socket):
        extractors, meta = data.get('extractors'), data.get('_meta')
        return self.save_data(['extractors'], data=extractors, socket=socket,
                              meta=meta)

    def save_items(self, data, socket):
        items, meta = data.get('items'), data.pop('_meta', None)
        return self.save_data(['items'], data=items, socket=socket, meta=meta)

    def save_data(self, path, type=None, data=None, socket=None, meta=None):
        if type is None:
            type = path[0]
        if any(v is None for v in (data, meta, socket)):
            raise BadRequest('No data provided')
        spec = socket.spec_manager.project_spec(meta['project'],
                                                socket.user.auth)
        try:
            obj = self.verify_data(path, data, spec)
        except (KeyError, IndexError) as ex:
            raise NotFound('"%s.json" could not be found' % '/'.join(path))
        except ValidationError as ex:
            print(('Not Valid: %s' % ex))
            raise BadRequest(str(ex))
        except BaseWSError as ex:
            print(('Other: %s' % ex))
            raise ex
        except Exception as ex:
            # XXX: Catch any other errors and log them leaving Websocket open
            log.err(traceback.format_exc(ex))
        else:
            try:
                spec.savejson(obj, [s.encode('utf-8') for s in path])
            except Exception as ex:
                # XXX: Catch errors in saving to the backend
                log.err(traceback.format_exc(ex))
            socket.update_spider(meta, **{type: obj})
            return obj

    def _decode(self, html, default=None):
        if default is None:
            default = []
        elif isinstance(default, six.string_types):
            default = [default]
        for encoding in itertools.chain(default, ('utf-8', 'windows-1252')):
            try:
                return html.decode(encoding)
            except UnicodeDecodeError:
                pass
        encoding = chardet.detect(html).get('encoding')
        return html.decode(encoding)

def update_project_data(data, socket):
    try:
        updater = ProjectData()
        meta = data['_meta'] or {}
        option = meta.get('type', '')
        command = getattr(updater, 'save_%s' % option)
    except KeyError:
        raise BadRequest('No metadata received')
    except AttributeError:
        raise BadRequest('Unknown option "%s" received' % option)
    else:
        resp = {'saved': {option: command(data, socket)}}
    id = meta.get('id')
    if id:
        resp['id'] = id
    resp.update(extract(socket))
    return resp

def delete_project_data(data, socket):
    """Use project and spider metadata along with a resource name to delete
    spiders and templates
    """
    try:
        meta = data['_meta']
        spec = socket.spec_manager.project_spec(meta['project'],
                                                socket.user.auth)
        option = meta.get('type', '')
        command = getattr(spec, 'remove_%s' % option)
        spider = meta['spider'].encode('utf-8')
        name = data.get('name').encode('utf-8')
    except KeyError:
        raise BadRequest('No or incorrect metadata received')
    except AttributeError:
        raise BadRequest('Unknown option "%s" received' % option)
    else:
        if option == 'spider':
            command(spider)
        else:
            command(spider, name)
            if socket.spiderspec:
                socket.open_spider(meta)
    resp = {'id': meta.get('id')}
    resp.update(extract(socket))
    return resp

def rename_project_data(data, socket):
    """Use project and spider metadata along with old and new names to rename
    spiders and templates
    """
    try:
        meta = data['_meta']
        spec = socket.spec_manager.project_spec(meta['project'],
                                                socket.user.auth)
        option = meta.get('type', '')
        command = getattr(spec, 'rename_%s' % option)
        spider = meta['spider'].encode('utf-8')
        old = data['old'].encode('utf-8')
        new = data['new'].encode('utf-8')
    except KeyError:
        raise BadRequest('No or incorrect metadata received')
    except AttributeError:
        raise BadRequest('Unknown option "%s" received' % option)
    else:
        if option == 'spider':
            command(old, new)
            if socket.spiderspec:
                socket.spiderspec.name = new
        else:
            command(spider, old, new)
            if socket.spiderspec:
                socket.open_spider(meta)
    return {'id': meta.get('id')}
<|endoftext|>


<commit_before>"""
Usage:
    smasherstats.py results [-s <tag>]... [-y <year>] [-y <year>] [options]
    smasherstats.py records [-s <tag>] [-s <tag>] [-y <year>] [-y <year>] [options]
    smasherstats.py settable [-s <tag>]... [-y <year>] [-y <year>] [options]
    smasherstats.py -h | --help

Get tournament results of specified smasher

Options:
  -h, --help                Show this help message and exit
  -s, --smasher <tag>       The tag of the smasher you want results for
  -i, --input_file <path>   Path to input file where tags are stored
  -o, --output_file <path>  Path to output file where results are put
  -t, --threshold <place>   Tournaments where the smasher placed worse will have
                            their names displayed
  -y, --year <year>         Specified year used to filter tournament dates
                            List 1 specific year or 2 to define a range
  -g, --game <game>         Specified game to get tournament results for
                            [default: Melee]
  -e, --event <event>       What event to pull results for
                            [default: Singles]
  --debug                   Run in debug mode
"""

import datetime
import sys

import requests
from prettytable import PrettyTable, ALL
from docopt import docopt
from bs4 import BeautifulSoup as bsoup
import pysmash

CURRENT_YEAR = datetime.datetime.now().year
smash = pysmash.SmashGG()

def nums_from_string(string):
    nums = ''
    for char in string:
        if char.isnumeric():
            nums += char
    return int(nums)

def getResults(tag, year):
    res = []
    smasher = '_'.join(i for i in tag.split())
    request = 'http://www.ssbwiki.com/Smasher:'
    page = requests.get(request + smasher)
    if page.status_code == 404:
        page = requests.get(request[:-8] + smasher)  # hehe
    soup = bsoup(page.content, "html.parser")
    while page.status_code == 404:
        print(f'Invalid tag \'{smasher}\'. Try again.')
        tag = input('Smasher: ')
        tag = ' '.join(i[0].upper() + i[1:] for i in tag.split())
        smasher = '_'.join(i for i in tag.split())
        page = requests.get(request + smasher)
        if page.status_code == 404:
            page = requests.get(request[:-8] + smasher)  # hehe
        soup = bsoup(page.content, "html.parser")

    tables = soup.find_all('div', {'id': 'mw-content-text'})[0].contents[2].contents[1].contents[1]
    for header in tables.find_all('h3'):
        if game in header.contents[0].text:
            tables = tables.contents[tables.index(header) + 2]
            break
    if str(year[0]).upper() == 'ALL':
        year = [int(tables.contents[3].contents[3].text.split(', ')[1]), CURRENT_YEAR]

    for i in range(3, len(tables.contents), 2):
        t = tables.contents[i]
        t_name = t.contents[1].text
        t_year = int((t.contents[3].text).strip(' ')[-4:])
        if event == 'Singles':
            t_place = str(t.contents[5].text).strip(' ')
        elif event == 'Doubles':
            t_place = str(t.contents[7].text).strip(' ')
        res += [[t_place, t_name, t_year]]
    res = [i for i in res if int(year[0]) <= i[2] <= int(year[-1])]
    return [res, year]

def getRecord(tags, results):
    if len(tags) == 2:
        print(f'{tags[0]} vs. {tags[1]}')
    tournaments = [r[1] for res in results for r in res[1] if res[0] in tags]
    filter_t = []
    for t in tournaments:
        if tournaments.count(t) == len(tags) and t not in filter_t:
            filter_t.append(t)
    tournaments = filter_t

    setcounts = [0, 0]
    gamecounts = [0, 0]

    fail_tournaments = []
    pt = PrettyTable()
    if len(tags) == 1:
        pt.field_names = ['Tournament', 'Round', f'{tags[0]} vs. ↓', 'Score', 'Outcome']
    elif len(tags) == 2:
        pt.field_names = ['Tournament', 'Round', f'{tags[0]} - {tags[1]}', 'Winner']
    pt_rows = []
    for tournament in tournaments:
        sys.stdout.write('\r')
        sys.stdout.write(f'Retrieving tournament {tournaments.index(tournament)+1}/{len(tournaments)}...')
        havePlayed = 0
        try:
            players = []
            b = 0
            tournament_name = '-'.join(tournament.replace('.', '').split())
            t = smash.tournament_show_event_brackets(tournament_name, 'melee-singles')
            while not set(map(str.lower, tags)).issubset([p['tag'].lower() for p in players]):
                b -= 1
                sets = smash.bracket_show_sets(t['bracket_ids'][b])
                players = smash.bracket_show_players(t['bracket_ids'][b])
        except:
            fail_tournaments.append(tournament)
            continue
        player_ids = ['' for _ in range(len(tags))]
        for p in players:
            if p['tag'] in tags:
                player_ids[tags.index(p['tag'])] = p['entrant_id']
        for s in sets:
            if all(str(n) != 'None' for n in s.values()):
                ids = [int(s['entrant_1_id']), int(s['entrant_2_id'])]
                scores = [s['entrant_1_score'], s['entrant_2_score']]
                if all(i in ids for i in player_ids):
                    havePlayed = 1
                    p_tag = ''
                    if len(tags) == 1:
                        p_id = ids[not ids.index(player_ids[0])]
                        for p in players:
                            if p['entrant_id'] == p_id:
                                p_tag = p['tag']
                    if ids[0] != player_ids[0]:
                        ids.reverse()
                        scores.reverse()
                    if scores[0] > scores[1]:
                        outcome = 'WIN'
                    else:
                        outcome = 'LOSS'
                    for i in range(len(gamecounts)):
                        gamecounts[i] += scores[i]
                    setcounts[scores.index(max(scores))] += 1
                    if len(tags) == 1:
                        res = [tournament,
                               s['full_round_text'],
                               p_tag,
                               f'{scores[0]} - {scores[1]}',
                               outcome]
                    elif len(tags) == 2:
                        res = [tournament,
                               s['full_round_text'],
                               f'{scores[0]} - {scores[1]}',
                               tags[player_ids.index(int(s['winner_id']))]]
                    pt_rows.append(res)
    for i in range(len(pt_rows)):
        row = pt_rows[i]
        r_name = row[0]
        if r_name == pt_rows[i-1][0]:
            r_name = ''
        elif i > 0:
            pt.add_row(['' for _ in range(len(pt.field_names))])
        pt.add_row([r_name] + row[1:])
    return pt, setcounts, gamecounts, fail_tournaments

# globals
smasher = ''
tags = []
threshold = 1
year = [CURRENT_YEAR]
game = 'Melee'
event = ''
input_file = ''
output_file = ''
valid = 0

args = docopt(__doc__)
if args['--debug']:
    print(args)
for arg in args:
    if args[arg] is not None:
        globals()[arg.strip('-')] = args[arg]

for y in year:
    if not str(y).isnumeric() and y.upper() != 'ALL':
        print('Invalid year \'{y}\'. Defaulting to current year.')
        y = CURRENT_YEAR
    y = str(y).lstrip('0')
    try:
        y = int(y)
    except:
        pass
if year == []:
    year = [CURRENT_YEAR]

games = [
    ['MELEE',
     'SMASH MELEE',
     'SMASH BROS MELEE',
     'Super Smash Bros. Melee',
     ''],

    ['64',
     'SMASH 64',
     'SUPER SMASH BROS 64',
     'Super Smash Bros.',
     ''],

    ['BRAWL',
     'SMASH BROS BRAWL',
     'SMASH BRAWL',
     'Super Smash Bros. Brawl',
     ''],

    ['SM4SH',
     'SMASH 4',
     'SMASH WII U',
     'SMASH BROS 4',
     'SMASH BROS WII U',
     'SMASH BROS 4',
     'SUPER SMASH BROS 4',
     'Super Smash Bros. for Wii U',
     ''],

    ['PM',
     'PROJECT MELEE',
     'SUPER SMASH BROS PROJECT M',
     'SUPER SMASH BROS PM',
     'Project M',
     '']
]
for g in games:
    if game.strip('.').upper() in g:
        game = g[-2]
        valid = 1
if not valid:
    print('Invalid game \'{game}\'. Defaulting to Melee.')
    game = 'Super Smash Bros. Melee'

events = [
    ['SINGLES',
     'Singles'],

    ['DOUBLES',
     'Doubles']
]

for e in events:
    if event.upper() in e:
        event = e[-1]
        valid = 1
if not valid:
    print('Invalid event \'{event}\'. Defaulting to Singles.')
    event = 'Singles'

if input_file != '':
    tags = [line.strip('\n') for line in open(input_file, 'r')]
if tags == [] and smasher == []:
    smasher = [input('Smasher: ')]
for tag in smasher:
    if tag != '' and tag.lower() not in map(str.lower, tags):
        tags += [tag]

results = []
tags = [' '.join(i[0].upper() + i[1:] for i in tag.split()) for tag in tags]
for tag in tags:
    r = getResults(tag, year)
    results.append([tag, r[0]])
    year = r[1]

if args['results']:
    for i in range(len(tags)):
        tag = tags[i]
        res = results[i][1]
        output = '-'*20 + '\n'
        output += f'{tag}\'s results for '
        if len(year) == 1:
            output += str(year[0])
        elif len(year) == 2:
            output += f'<{year[0]}, {year[1]}>'
        output += ':'
        if int(threshold) not in [0, 1]:
            output += f'\nTournament names listed for placings of {threshold} or below.\n'

        res = [r for r in res if any(c.isdigit() for c in r[0])]
        res = sorted(res, key=lambda x: nums_from_string(x[0]))

        # sorted by place
        # formatted like so: [[place, name, year], ...]
        for i in range(len(res)):
            r = [i[0] for i in res]
            place = res[i][0]
            if res[i - 1][0] != place:
                output += '\n'
                count = r.count(place)
                t_str = f'{place} - {count}'
                if nums_from_string(place) >= int(threshold) > 0:
                    for k in range(len(res)):
                        if res[k][0] == place:
                            t_name = res[k][1]
                            t_year = str(res[k][2])
                            if t_str[0] != '\n':
                                t_str = '\n' + t_str
                            t_str += '\n - ' + t_name + ' '
                            if len(year) != 1 and t_year not in t_name:
                                t_str += f'({t_year})'
                output += t_str
        if i == len(tags) - 1:
            output += '-'*20 + '\n'
        if output_file == '':
            print(output)
        else:
            with open(output_file, 'a+') as f:
                ofile = output_file.replace('\\', ' ').replace('/', ' ').split()[-1]
                if output not in open(output_file).read():
                    f.write(output)
                    print(f'{tag} written to {ofile}')
                else:
                    print(f'{tag} already in {ofile}')

if args['records']:
    record = getRecord(tags, results)
    print('\n\nTournaments where specified players were present but results failed to be retrieved:')
    for f in record[3]:
        print(f' - {f}')
    print()
    print(record[0], '\n')
    if len(tags) == 2:
        print(f'Total Set Count: {tags[0]} {record[1][0]} - {record[1][1]} {tags[1]}')
        print(f'Total Game Count: {tags[0]} {record[2][0]} - {record[2][1]} {tags[1]}')

if args['settable']:
    settable = PrettyTable(hrules=ALL)
    settable.field_names = ['↓ vs. →'] + tags
    st = [['-' for _ in range(len(tags))] for _ in range(len(tags))]
    for i in range(len(tags)):
        for j in range(i+1, len(tags)):
            record = getRecord([tags[i], tags[j]], results)[1]
            print('\n')
            st[i][j] = f'{record[0]} - {record[1]}'
            st[j][i] = f'{record[1]} - {record[0]}'
    for i in range(len(st)):
        settable.add_row([tags[i]] + st[i])
    print(settable)
<commit_msg>cleanup<commit_after>"""
Usage:
    smasherstats.py results [-s <tag>]... [-y <year>] [-y <year>] [options]
    smasherstats.py records [-s <tag>] [-s <tag>] [-y <year>] [-y <year>] [options]
    smasherstats.py settable [-s <tag>]... [-y <year>] [-y <year>] [options]
    smasherstats.py -h | --help

Get tournament results of specified smasher

Options:
  -h, --help                Show this help message and exit
  -s, --smasher <tag>       The tag of the smasher you want results for
  -i, --input_file <path>   Path to input file where tags are stored
  -o, --output_file <path>  Path to output file where results are put
  -t, --threshold <place>   Tournaments where the smasher placed worse will have
                            their names displayed
  -y, --year <year>         Specified year used to filter tournament dates
                            List 1 specific year or 2 to define a range
  -g, --game <game>         Specified game to get tournament results for
                            [default: Melee]
  -e, --event <event>       What event to pull results for
                            [default: Singles]
  --debug                   Run in debug mode
"""

import datetime
import sys

import requests
from prettytable import PrettyTable, ALL
from docopt import docopt
from bs4 import BeautifulSoup as bsoup
import pysmash

CURRENT_YEAR = datetime.datetime.now().year
smash = pysmash.SmashGG()

def nums_from_string(string):
    nums = ''
    for char in string:
        if char.isnumeric():
            nums += char
    return int(nums)

def getResults(tag, year):
    res = []
    smasher = '_'.join(i for i in tag.split())
    request = 'http://www.ssbwiki.com/Smasher:'
    page = requests.get(request + smasher)
    if page.status_code == 404:
        page = requests.get(request[:-8] + smasher)  # hehe
    soup = bsoup(page.content, "html.parser")
    while page.status_code == 404:
        print(f'Invalid tag \'{smasher}\'. Try again.')
        tag = input('Smasher: ')
        tag = ' '.join(i[0].upper() + i[1:] for i in tag.split())
        smasher = '_'.join(i for i in tag.split())
        page = requests.get(request + smasher)
        if page.status_code == 404:
            page = requests.get(request[:-8] + smasher)  # hehe
        soup = bsoup(page.content, "html.parser")

    tables = soup.find_all('div', {'id': 'mw-content-text'})[0].contents[2].contents[1].contents[1]
    for header in tables.find_all('h3'):
        if game in header.contents[0].text:
            tables = tables.contents[tables.index(header) + 2]
            break
    if str(year[0]).upper() == 'ALL':
        year = [int(tables.contents[3].contents[3].text.split(', ')[1]), CURRENT_YEAR]

    for i in range(3, len(tables.contents), 2):
        t = tables.contents[i]
        t_name = t.contents[1].text
        t_year = int((t.contents[3].text).strip(' ')[-4:])
        if event == 'Singles':
            t_place = str(t.contents[5].text).strip(' ')
        elif event == 'Doubles':
            t_place = str(t.contents[7].text).strip(' ')
        res += [[t_place, t_name, t_year]]
    res = [i for i in res if int(year[0]) <= i[2] <= int(year[-1])]
    return [res, year]

def getRecord(tags, results):
    if len(tags) == 2:
        print(f'{tags[0]} vs. {tags[1]}')
    tournaments = [r[1] for res in results for r in res[1] if res[0] in tags]
    filter_t = []
    for t in tournaments:
        if tournaments.count(t) == len(tags) and t not in filter_t:
            filter_t.append(t)
    tournaments = filter_t

    setcounts = [0, 0]
    gamecounts = [0, 0]

    fail_tournaments = []
    pt = PrettyTable()
    if len(tags) == 1:
        pt.field_names = ['Tournament', 'Round', f'{tags[0]} vs. ↓', 'Score', 'Outcome']
    elif len(tags) == 2:
        pt.field_names = ['Tournament', 'Round', f'{tags[0]} - {tags[1]}', 'Winner']
    pt_rows = []
    for tournament in tournaments:
        sys.stdout.write('\r')
        sys.stdout.write(f'Retrieving tournament {tournaments.index(tournament)+1}/{len(tournaments)}...')
        sys.stdout.flush()
        havePlayed = 0
        try:
            players = []
            b = 0
            tournament_name = '-'.join(tournament.replace('.', '').split())
            t = smash.tournament_show_event_brackets(tournament_name, 'melee-singles')
            while not set(map(str.lower, tags)).issubset([p['tag'].lower() for p in players]):
                b -= 1
                sets = smash.bracket_show_sets(t['bracket_ids'][b])
                players = smash.bracket_show_players(t['bracket_ids'][b])
        except:
            fail_tournaments.append(tournament)
            continue
        player_ids = ['' for _ in range(len(tags))]
        for p in players:
            if p['tag'] in tags:
                player_ids[tags.index(p['tag'])] = p['entrant_id']
        for s in sets:
            if all(str(n) != 'None' for n in s.values()):
                ids = [int(s['entrant_1_id']), int(s['entrant_2_id'])]
                scores = [s['entrant_1_score'], s['entrant_2_score']]
                if all(i in ids for i in player_ids):
                    havePlayed = 1
                    p_tag = ''
                    if len(tags) == 1:
                        p_id = ids[not ids.index(player_ids[0])]
                        for p in players:
                            if p['entrant_id'] == p_id:
                                p_tag = p['tag']
                    if ids[0] != player_ids[0]:
                        ids.reverse()
                        scores.reverse()
                    if scores[0] > scores[1]:
                        outcome = 'WIN'
                    else:
                        outcome = 'LOSS'
                    for i in range(len(gamecounts)):
                        gamecounts[i] += scores[i]
                    setcounts[scores.index(max(scores))] += 1
                    if len(tags) == 1:
                        res = [tournament,
                               s['full_round_text'],
                               p_tag,
                               f'{scores[0]} - {scores[1]}',
                               outcome]
                    elif len(tags) == 2:
                        res = [tournament,
                               s['full_round_text'],
                               f'{scores[0]} - {scores[1]}',
                               tags[player_ids.index(int(s['winner_id']))]]
                    pt_rows.append(res)
    for i in range(len(pt_rows)):
        row = pt_rows[i]
        r_name = row[0]
        if r_name == pt_rows[i-1][0]:
            r_name = ''
        elif i > 0:
            pt.add_row(['' for _ in range(len(pt.field_names))])
        pt.add_row([r_name] + row[1:])
    return pt, setcounts, gamecounts, fail_tournaments

# globals
smasher = ''
tags = []
threshold = 1
year = [CURRENT_YEAR]
game = 'Melee'
event = ''
input_file = ''
output_file = ''
valid = 0

args = docopt(__doc__)
if args['--debug']:
    print(args)
for arg in args:
    if args[arg] is not None:
        globals()[arg.strip('-')] = args[arg]

for y in year:
    if not str(y).isnumeric() and y.upper() != 'ALL':
        print('Invalid year \'{y}\'. Defaulting to current year.')
        y = CURRENT_YEAR
    y = str(y).lstrip('0')
    try:
        y = int(y)
    except:
        pass
if year == []:
    year = [CURRENT_YEAR]

games = [
    ['MELEE',
     'SMASH MELEE',
     'SMASH BROS MELEE',
     'Super Smash Bros. Melee',
     ''],

    ['64',
     'SMASH 64',
     'SUPER SMASH BROS 64',
     'Super Smash Bros.',
     ''],

    ['BRAWL',
     'SMASH BROS BRAWL',
     'SMASH BRAWL',
     'Super Smash Bros. Brawl',
     ''],

    ['SM4SH',
     'SMASH 4',
     'SMASH WII U',
     'SMASH BROS 4',
     'SMASH BROS WII U',
     'SMASH BROS 4',
     'SUPER SMASH BROS 4',
     'Super Smash Bros. for Wii U',
     ''],

    ['PM',
     'PROJECT MELEE',
     'SUPER SMASH BROS PROJECT M',
     'SUPER SMASH BROS PM',
     'Project M',
     '']
]
for g in games:
    if game.strip('.').upper() in g:
        game = g[-2]
        valid = 1
if not valid:
    print('Invalid game \'{game}\'. Defaulting to Melee.')
    game = 'Super Smash Bros. Melee'

events = [
    ['SINGLES',
     'Singles'],

    ['DOUBLES',
     'Doubles']
]

for e in events:
    if event.upper() in e:
        event = e[-1]
        valid = 1
if not valid:
    print('Invalid event \'{event}\'. Defaulting to Singles.')
    event = 'Singles'

if input_file != '':
    tags = [line.strip('\n') for line in open(input_file, 'r')]
if tags == [] and smasher == []:
    smasher = [input('Smasher: ')]
for tag in smasher:
    if tag != '' and tag.lower() not in map(str.lower, tags):
        tags += [tag]

results = []
tags = [' '.join(i[0].upper() + i[1:] for i in tag.split()) for tag in tags]
for tag in tags:
    r = getResults(tag, year)
    results.append([tag, r[0]])
    year = r[1]

if args['results']:
    for i in range(len(tags)):
        tag = tags[i]
        res = results[i][1]
        output = '-'*20 + '\n'
        output += f'{tag}\'s results for '
        if len(year) == 1:
            output += str(year[0])
        elif len(year) == 2:
            output += f'<{year[0]}, {year[1]}>'
        output += ':'
        if int(threshold) not in [0, 1]:
            output += f'\nTournament names listed for placings of {threshold} or below.\n'

        res = [r for r in res if any(c.isdigit() for c in r[0])]
        res = sorted(res, key=lambda x: nums_from_string(x[0]))

        # sorted by place
        # formatted like so: [[place, name, year], ...]
        for i in range(len(res)):
            r = [i[0] for i in res]
            place = res[i][0]
            if res[i - 1][0] != place:
                output += '\n'
                count = r.count(place)
                t_str = f'{place} - {count}'
                if nums_from_string(place) >= int(threshold) > 0:
                    for k in range(len(res)):
                        if res[k][0] == place:
                            t_name = res[k][1]
                            t_year = str(res[k][2])
                            if t_str[0] != '\n':
                                t_str = '\n' + t_str
                            t_str += '\n - ' + t_name + ' '
                            if len(year) != 1 and t_year not in t_name:
                                t_str += f'({t_year})'
                output += t_str
        if i == len(tags) - 1:
            output += '-'*20 + '\n'
        if output_file == '':
            print(output)
        else:
            with open(output_file, 'a+') as f:
                ofile = output_file.replace('\\', ' ').replace('/', ' ').split()[-1]
                if output not in open(output_file).read():
                    f.write(output)
                    print(f'{tag} written to {ofile}')
                else:
                    print(f'{tag} already in {ofile}')

if args['records']:
    record = getRecord(tags, results)
    print('\n\nTournaments where specified players were present but results failed to be retrieved:')
    for f in record[3]:
        print(f' - {f}')
    print()
    print(record[0], '\n')
    if len(tags) == 2:
        print(f'Total Set Count: {tags[0]} {record[1][0]} - {record[1][1]} {tags[1]}')
        print(f'Total Game Count: {tags[0]} {record[2][0]} - {record[2][1]} {tags[1]}')

if args['settable']:
    settable = PrettyTable(hrules=ALL)
    settable.field_names = ['↓ vs. →'] + tags
    st = [['-' for _ in range(len(tags))] for _ in range(len(tags))]
    for i in range(len(tags)):
        for j in range(i+1, len(tags)):
            record = getRecord([tags[i], tags[j]], results)[1]
            print('\n')
            st[i][j] = f'{record[0]} - {record[1]}'
            st[j][i] = f'{record[1]} - {record[0]}'
    for i in range(len(st)):
        settable.add_row([tags[i]] + st[i])
    print(settable)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
makes modifications to the controlled vocabulary (implemented as
ckanext-scheming "choices")
HvW - 2016-06-07
"""

import ckanapi
import argparse
import sys
import json
import pprint
import os

LOCAL_SCHEMA=("/usr/lib/ckan/default/src/ckanext-eaw_schema/ckanext/" +
               "eaw_schema/eaw_schema_default.json")

def mkparser():
    parser = argparse.ArgumentParser(description=
                                     "Make modifications to the controlled "+
                                     "vocabulary (implemented as "+
                                     "ckanext-scheming \"choices\"")
    parser.add_argument('field', help='the schema field to be modified')
    parser.add_argument('--del', action='store_true', help='delete terms '+
                        '(default is adding terms)')
    parser.add_argument('--resource', action='store_true', help='action ' +
                        'refers to resource field (default is dataset field)')
    parser.add_argument('terms', nargs='+', help='the terms to be added '
                        +'(removed). Have the format "value,label" for adding,' +
                        ' and "value" for removing')
    return(parser)

def postparse(params, parser):
    terms = params['terms']
    terms = [tuple(x.split(',')) for x in terms]
    if params['del'] and not all([len(x) == 1 for x in terms]):
        parser.print_help()
        sys.exit(1)
    elif not params['del'] and not all([len(x) == 2 for x in terms]):
        parser.print_help()
        sys.exit(1)
    terms = [x[0] if len(x) == 1 else x for x in terms]
    return(terms)

def load_schema(schemafile):
    try:
        with open(schemafile) as sf:
            schema = json.load(sf)
    except ValueError:
        raise(SystemExit("Schema file {} doesn't parse into JSON"
                         .format(schemafile), 1))
    except IOError:
        raise(SystemExit("Could not open schema file 'testschema'", 1))
    return(schema)

def check_unique(field, choices, terms):
    for t in [x[0] for x in terms]:
        if t in [x['value'] for x in choices]:
            raise SystemExit('{} already in {}'.format(t, field))

def update_field(schema, typ, field, remove, terms):
    " typ: 'dataset_field' or 'resource_field'"
    def _build_choices(terms):
        ch = [{'value': t[0], 'label': t[1]} for t in terms]
        return ch

    def _get_val_index(val, choices):
        idx = [x.get('value') for x in choices].index(val)
        return(idx)
    try:
        f = [x for x in schema[typ] if x["field_name"] == field]
    except KeyError:
        raise SystemExit('Could not find field_type "{}"'.format(typ))
    if not f:
        raise SystemExit("Could not find field \"{}\" in \"{}\""
                         .format(field, typ))
    assert(len(f) == 1)
    c = f[0]['choices']
    if not remove:
        check_unique(field, c, terms)
        c.extend(_build_choices(terms))
    else:
        try:
            rmidx = [_get_val_index(val, c) for val in terms]
        except ValueError:
            raise SystemExit('Not all terms found in ' +
                             'field "{}" in "{}"'.format(field, typ))
        if len(rmidx) < len(terms):
            raise SystemExit('Not all terms found in ' +
                 'field "{}" in "{}"'.format(field, typ))
        cnew = [x[1] for x in enumerate(c) if x[0] not in rmidx]
        f[0]['choices'] = cnew
    return(schema)

def write_schema(newschema, path):
    with open(path, 'w') as f:
        json.dump(newschema, f, indent=2)

def main():
    parser = mkparser()
    params = vars(parser.parse_args())
    terms = postparse(params, parser)
    field = params['field']
    remove = params['del']
    typ = 'resource_fields' if params['resource'] else 'dataset_fields'
    schema = load_schema(LOCAL_SCHEMA)
    newschema = update_field(schema, typ, field, remove, terms)
    write_schema(newschema, LOCAL_SCHEMA)

if __name__ == '__main__':
    main()

<commit_msg>fixed help test<commit_after>#!/usr/bin/env python

"""
makes modifications to the controlled vocabulary (implemented as
ckanext-scheming "choices")
HvW - 2016-06-07
"""

import ckanapi
import argparse
import sys
import json
import pprint
import os

LOCAL_SCHEMA=("/usr/lib/ckan/default/src/ckanext-eaw_schema/ckanext/" +
               "eaw_schema/eaw_schema_default.json")

def mkparser():
    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description=
"""
Make modifications to the controlled vocabulary \"field\"
(implemented as ckanext-scheming "choices").
""",                                  epilog=
"""
Examples:
ck_choices.py variables "new_var_1,New Variable One" newvar2,"Another One"
adds two new terms to the dataset_field "variables".

ck_choices.py variables --del new_var_1 newvar2
deletes them.
"""
    )
    parser.add_argument('field', help='the schema field to be modified',
                        metavar='FIELD')
    parser.add_argument('--del', action='store_true', help='delete terms '+
                        '(default is adding terms)')
    parser.add_argument('--resource', action='store_true', help='action ' +
                        'refers to resource field (default is dataset field)')
    parser.add_argument('terms', nargs='+', help='the terms to be added '
                        +'(removed). Have the format "value,label" for adding,' +
                        ' and "value" for removing', metavar='TERM')
    return(parser)

def postparse(params, parser):
    terms = params['terms']
    terms = [tuple(x.split(',')) for x in terms]
    if params['del'] and not all([len(x) == 1 for x in terms]):
        parser.print_help()
        sys.exit(1)
    elif not params['del'] and not all([len(x) == 2 for x in terms]):
        parser.print_help()
        sys.exit(1)
    terms = [x[0] if len(x) == 1 else x for x in terms]
    return(terms)

def load_schema(schemafile):
    try:
        with open(schemafile) as sf:
            schema = json.load(sf)
    except ValueError:
        raise(SystemExit("Schema file {} doesn't parse into JSON"
                         .format(schemafile), 1))
    except IOError:
        raise(SystemExit("Could not open schema file 'testschema'", 1))
    return(schema)

def check_unique(field, choices, terms):
    for t in [x[0] for x in terms]:
        if t in [x['value'] for x in choices]:
            raise SystemExit('{} already in {}'.format(t, field))

def update_field(schema, typ, field, remove, terms):
    " typ: 'dataset_field' or 'resource_field'"
    def _build_choices(terms):
        ch = [{'value': t[0], 'label': t[1]} for t in terms]
        return ch

    def _get_val_index(val, choices):
        idx = [x.get('value') for x in choices].index(val)
        return(idx)
    try:
        f = [x for x in schema[typ] if x["field_name"] == field]
    except KeyError:
        raise SystemExit('Could not find field_type "{}"'.format(typ))
    if not f:
        raise SystemExit("Could not find field \"{}\" in \"{}\""
                         .format(field, typ))
    assert(len(f) == 1)
    c = f[0]['choices']
    if not remove:
        check_unique(field, c, terms)
        c.extend(_build_choices(terms))
    else:
        try:
            rmidx = [_get_val_index(val, c) for val in terms]
        except ValueError:
            raise SystemExit('Not all terms found in ' +
                             'field "{}" in "{}"'.format(field, typ))
        if len(rmidx) < len(terms):
            raise SystemExit('Not all terms found in ' +
                 'field "{}" in "{}"'.format(field, typ))
        cnew = [x[1] for x in enumerate(c) if x[0] not in rmidx]
        f[0]['choices'] = cnew
    return(schema)

def write_schema(newschema, path):
    with open(path, 'w') as f:
        json.dump(newschema, f, indent=2)

def main():
    parser = mkparser()
    params = vars(parser.parse_args())
    terms = postparse(params, parser)
    field = params['field']
    remove = params['del']
    typ = 'resource_fields' if params['resource'] else 'dataset_fields'
    schema = load_schema(LOCAL_SCHEMA)
    newschema = update_field(schema, typ, field, remove, terms)
    write_schema(newschema, LOCAL_SCHEMA)

if __name__ == '__main__':
    main()

<|endoftext|>