<commit_before>from pyramid_celery import celery_app as app

from owslib.wps import WebProcessingService, monitorExecution, ComplexDataInput
from owslib.util import build_get_url
import json
import yaml
import uuid
import urllib
from urlparse import urlparse
from datetime import datetime
from birdfeeder import feed_from_thredds, clear

from phoenix.db import mongodb
from phoenix.security import generate_access_token

from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

def task_result(task_id):
    return app.AsyncResult(task_id)

def log(job):
    log_msg = '{0:3d}%: {1}'.format(job.get('progress'), job.get('status_message'))
    if not 'log' in job:
        job['log'] = []
    # skip same log messages
    if len(job['log']) == 0 or job['log'][-1] != log_msg:
        job['log'].append(log_msg)
        logger.info(log_msg)

def log_error(job, error):
    log_msg = 'ERROR: {0.text} - code={0.code} - locator={0.locator}'.format(error)
    if not 'log' in job:
        job['log'] = []
    # skip same log messages
    if len(job['log']) == 0 or job['log'][-1] != log_msg:
        job['log'].append(log_msg)
        logger.error(log_msg)

def add_job(db, userid, task_id, service, title, abstract, status_location, is_workflow=False, caption=None):
    tags = ['dev']
    if is_workflow:
        tags.append('workflow')
    else:
        tags.append('single')
    job = dict(
        identifier = str(uuid.uuid1()),
        task_id = task_id,
        userid = userid,
        is_workflow = is_workflow,
        service = service,
        title = title,
        abstract = abstract,
        status_location = status_location,
        created = datetime.now(),
        is_complete = False,
        tags = tags,
        caption = caption,
        status = "ProcessAccepted",
        )
    db.jobs.insert(job)
    return job

def get_access_token(userid):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    # update access token
    generate_access_token(registry, userid)

    user = db.users.find_one(dict(identifier=userid))
    return user.get('twitcher_token')

def wps_headers(userid):
    headers = {}
    access_token = get_access_token(userid)
    if access_token is not None:
        headers['Access-Token'] = access_token
    return headers

@app.task(bind=True)
def esgf_logon(self, userid, url, openid, password):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    inputs = []
    inputs.append( ('openid', openid) )
    inputs.append( ('password', password) )
    outputs = [('output',True),('expires',False)]

    wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=wps_headers(userid))
    execution = wps.execute(identifier="esgf_logon", inputs=inputs, output=outputs)
    monitorExecution(execution)

    if execution.isSucceded():
        credentials = execution.processOutputs[0].reference
        cert_expires = execution.processOutputs[1].data[0]
        user = db.users.find_one({'identifier':userid})
        user['openid'] = openid
        user['credentials'] = credentials
        user['cert_expires'] = cert_expires
        db.users.update({'identifier':userid}, user)
    return execution.status

@app.task(bind=True)
def execute_workflow(self, userid, url, workflow, caption=None):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    # generate and run dispel workflow
    # TODO: fix owslib wps for unicode/yaml parameters
    logger.debug('workflow=%s', workflow)
    headers=wps_headers(userid)
    # TODO: handle access token in workflow
    workflow['worker']['url'] = build_get_url(workflow['worker']['url'], {'access_token': headers.get('Access-Token', '') })
    logger.debug('workflow_mod=%s', workflow)
    inputs=[('workflow', ComplexDataInput(json.dumps(workflow), mimeType="text/yaml", encoding="UTF-8") )]
    logger.debug('inputs=%s', inputs)
    outputs=[('output', True), ('logfile', True)]

    wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers)
    worker_wps = WebProcessingService(url=workflow['worker']['url'], skip_caps=False, verify=False)
    execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs, lineage=True)

    job = add_job(db, userid,
                  task_id = self.request.id,
                  is_workflow = True,
                  service = worker_wps.identification.title,
                  title = workflow['worker']['identifier'],
                  abstract = '',
                  caption = caption,
                  status_location = execution.statusLocation)

    while execution.isNotComplete():
        try:
            execution.checkStatus(sleepSecs=3)
            job['status'] = execution.getStatus()
            job['status_message'] = execution.statusMessage
            job['is_complete'] = execution.isComplete()
            job['is_succeded'] = execution.isSucceded()
            job['progress'] = execution.percentCompleted
            duration = datetime.now() - job.get('created', datetime.now())
            job['duration'] = str(duration).split('.')[0]
            if execution.isComplete():
                job['finished'] = datetime.now()
                if execution.isSucceded():
                    for output in execution.processOutputs:
                        if 'output' == output.identifier:
                            result = yaml.load(urllib.urlopen(output.reference))
                            job['worker_status_location'] = result['worker']['status_location']
                    job['progress'] = 100
                    log(job)
                else:
                    job['status_message'] = '\n'.join(error.text for error in execution.errors)
                    for error in execution.errors:
                        log_error(job, error)
            else:
                log(job)
        except:
            logger.exception("Could not read status xml document.")
        else:
            db.jobs.update({'identifier': job['identifier']}, job)
    return execution.getStatus()

@app.task(bind=True)
def execute_process(self, userid, url, identifier, inputs, outputs, caption=None):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    wps = WebProcessingService(url=url, skip_caps=False, verify=False, headers=wps_headers(userid))
    execution = wps.execute(identifier, inputs=inputs, output=outputs, lineage=True)
    job = add_job(db, userid,
                  task_id = self.request.id,
                  is_workflow = False,
                  service = wps.identification.title,
                  title = execution.process.identifier,
                  abstract = getattr(execution.process, "abstract", ""),
                  caption = caption,
                  status_location = execution.statusLocation)

    while execution.isNotComplete():
        try:
            execution.checkStatus(sleepSecs=3)
            job['status'] = execution.getStatus()
            job['status_message'] = execution.statusMessage
            job['is_complete'] = execution.isComplete()
            job['is_succeded'] = execution.isSucceded()
            job['progress'] = execution.percentCompleted
            duration = datetime.now() - job.get('created', datetime.now())
            job['duration'] = str(duration).split('.')[0]
            if execution.isComplete():
                job['finished'] = datetime.now()
                if execution.isSucceded():
                    job['progress'] = 100
                    log(job)
                else:
                    job['status_message'] = '\n'.join(error.text for error in execution.errors)
                    for error in execution.errors:
                        log_error(job, error)
            else:
                log(job)
        except:
            logger.exception("Could not read status xml document.")
        else:
            db.jobs.update({'identifier': job['identifier']}, job)
    return execution.getStatus()

@app.task(bind=True)
def index_thredds(self, url, maxrecords=-1, depth=2):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)
    task = dict(task_id=self.request.id, url=url, status='started')
    db.tasks.save(task)
    service = registry.settings.get('solr.url')
    try:
        feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000)
        task['status'] = 'success'
    except:
        task['status'] = 'failure'
        raise
    finally:
        db.tasks.update({'url': task['url']}, task)

@app.task(bind=True)
def clear_index(self):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)
    service = registry.settings.get('solr.url')
    clear(service=service)
    db.tasks.drop()
<commit_msg>prevent error in job task<commit_after>from pyramid_celery import celery_app as app

from owslib.wps import WebProcessingService, monitorExecution, ComplexDataInput
from owslib.util import build_get_url
import json
import yaml
import uuid
import urllib
from urlparse import urlparse
from datetime import datetime
from birdfeeder import feed_from_thredds, clear

from phoenix.db import mongodb
from phoenix.security import generate_access_token

from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

def task_result(task_id):
    return app.AsyncResult(task_id)

def log(job):
    log_msg = '{0:3d}%: {1}'.format(job.get('progress'), job.get('status_message'))
    if not 'log' in job:
        job['log'] = []
    # skip same log messages
    if len(job['log']) == 0 or job['log'][-1] != log_msg:
        job['log'].append(log_msg)
        logger.info(log_msg)

def log_error(job, error):
    log_msg = 'ERROR: {0.text} - code={0.code} - locator={0.locator}'.format(error)
    if not 'log' in job:
        job['log'] = []
    # skip same log messages
    if len(job['log']) == 0 or job['log'][-1] != log_msg:
        job['log'].append(log_msg)
        logger.error(log_msg)

def add_job(db, userid, task_id, service, title, abstract, status_location, is_workflow=False, caption=None):
    tags = ['dev']
    if is_workflow:
        tags.append('workflow')
    else:
        tags.append('single')
    job = dict(
        identifier = str(uuid.uuid1()),
        task_id = task_id,
        userid = userid,
        is_workflow = is_workflow,
        service = service,
        title = title,
        abstract = abstract,
        status_location = status_location,
        created = datetime.now(),
        is_complete = False,
        tags = tags,
        caption = caption,
        status = "ProcessAccepted",
        )
    db.jobs.insert(job)
    return job

def get_access_token(userid):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    # update access token
    generate_access_token(registry, userid)

    user = db.users.find_one(dict(identifier=userid))
    return user.get('twitcher_token')

def wps_headers(userid):
    headers = {}
    access_token = get_access_token(userid)
    if access_token is not None:
        headers['Access-Token'] = access_token
    return headers

@app.task(bind=True)
def esgf_logon(self, userid, url, openid, password):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    inputs = []
    inputs.append( ('openid', openid) )
    inputs.append( ('password', password) )
    outputs = [('output',True),('expires',False)]

    wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=wps_headers(userid))
    execution = wps.execute(identifier="esgf_logon", inputs=inputs, output=outputs)
    monitorExecution(execution)

    if execution.isSucceded():
        credentials = execution.processOutputs[0].reference
        cert_expires = execution.processOutputs[1].data[0]
        user = db.users.find_one({'identifier':userid})
        user['openid'] = openid
        user['credentials'] = credentials
        user['cert_expires'] = cert_expires
        db.users.update({'identifier':userid}, user)
    return execution.status

@app.task(bind=True)
def execute_workflow(self, userid, url, workflow, caption=None):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    # generate and run dispel workflow
    # TODO: fix owslib wps for unicode/yaml parameters
    logger.debug('workflow=%s', workflow)
    headers=wps_headers(userid)
    # TODO: handle access token in workflow
    workflow['worker']['url'] = build_get_url(workflow['worker']['url'], {'access_token': headers.get('Access-Token', '') })
    logger.debug('workflow_mod=%s', workflow)
    inputs=[('workflow', ComplexDataInput(json.dumps(workflow), mimeType="text/yaml", encoding="UTF-8") )]
    logger.debug('inputs=%s', inputs)
    outputs=[('output', True), ('logfile', True)]

    wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers)
    worker_wps = WebProcessingService(url=workflow['worker']['url'], skip_caps=False, verify=False)
    execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs, lineage=True)

    job = add_job(db, userid,
                  task_id = self.request.id,
                  is_workflow = True,
                  service = worker_wps.identification.title,
                  title = workflow['worker']['identifier'],
                  abstract = '',
                  caption = caption,
                  status_location = execution.statusLocation)

    while execution.isNotComplete():
        try:
            execution.checkStatus(sleepSecs=3)
            job['status'] = execution.getStatus()
            job['status_message'] = execution.statusMessage
            job['is_complete'] = execution.isComplete()
            job['is_succeded'] = execution.isSucceded()
            job['progress'] = execution.percentCompleted
            duration = datetime.now() - job.get('created', datetime.now())
            job['duration'] = str(duration).split('.')[0]
            if execution.isComplete():
                job['finished'] = datetime.now()
                if execution.isSucceded():
                    for output in execution.processOutputs:
                        if 'output' == output.identifier:
                            result = yaml.load(urllib.urlopen(output.reference))
                            job['worker_status_location'] = result['worker']['status_location']
                    job['progress'] = 100
                    log(job)
                else:
                    job['status_message'] = '\n'.join(error.text for error in execution.errors)
                    for error in execution.errors:
                        log_error(job, error)
            else:
                log(job)
        except:
            logger.exception("Could not read status xml document.")
        else:
            db.jobs.update({'identifier': job['identifier']}, job)
    return execution.getStatus()

@app.task(bind=True)
def execute_process(self, userid, url, identifier, inputs, outputs, caption=None):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)

    wps = WebProcessingService(url=url, skip_caps=False, verify=False, headers=wps_headers(userid))
    execution = wps.execute(identifier, inputs=inputs, output=outputs, lineage=True)
    job = add_job(db, userid,
                  task_id = self.request.id,
                  is_workflow = False,
                  service = wps.identification.title,
                  title = identifier,
                  abstract = getattr(execution.process, "abstract", ""),
                  caption = caption,
                  status_location = execution.statusLocation)

    while execution.isNotComplete():
        try:
            execution.checkStatus(sleepSecs=3)
            job['status'] = execution.getStatus()
            job['status_message'] = execution.statusMessage
            job['is_complete'] = execution.isComplete()
            job['is_succeded'] = execution.isSucceded()
            job['progress'] = execution.percentCompleted
            duration = datetime.now() - job.get('created', datetime.now())
            job['duration'] = str(duration).split('.')[0]
            if execution.isComplete():
                job['finished'] = datetime.now()
                if execution.isSucceded():
                    job['progress'] = 100
                    log(job)
                else:
                    job['status_message'] = '\n'.join(error.text for error in execution.errors)
                    for error in execution.errors:
                        log_error(job, error)
            else:
                log(job)
        except:
            logger.exception("Could not read status xml document.")
        else:
            db.jobs.update({'identifier': job['identifier']}, job)
    return execution.getStatus()

@app.task(bind=True)
def index_thredds(self, url, maxrecords=-1, depth=2):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)
    task = dict(task_id=self.request.id, url=url, status='started')
    db.tasks.save(task)
    service = registry.settings.get('solr.url')
    try:
        feed_from_thredds(service=service, catalog_url=url, depth=depth, maxrecords=maxrecords, batch_size=50000)
        task['status'] = 'success'
    except:
        task['status'] = 'failure'
        raise
    finally:
        db.tasks.update({'url': task['url']}, task)

@app.task(bind=True)
def clear_index(self):
    registry = app.conf['PYRAMID_REGISTRY']
    db = mongodb(registry)
    service = registry.settings.get('solr.url')
    clear(service=service)
    db.tasks.drop()
<|endoftext|>


<commit_before>import copy
import json
import logging
import os
import plistlib
import re
import sublime
import yaml
from .functions import snake_to_camel, camel_to_snake

ST_SYNTAX_FILE_EXTS = [".sublime-syntax", ".tmLanguage"]

class SyntaxMappings(object):
    settings = None
    logger = None

    # items of this list are dicts whose keys are
    #     rule_source: str
    #     file_extensions: list[str]
    #     file_path: str
    #     first_line_match: list[str]
    #     first_line_match_compiled: list[compiled regex object]
    syntax_mappings = []

    # the path of all syntax files
    syntax_files = []

    def __init__(self, settings: sublime.Settings, logger: logging.Logger) -> None:
        self.settings = settings
        self.logger = logger

        self.syntax_files = self._find_syntax_file_paths()
        self.syntax_mappings = self._build_syntax_mappings()

        self.logger.debug("Found syntax files: {}".format(self.syntax_files))

    def __iter__(self):
        return iter(self.value())

    def __len__(self) -> int:
        return len(self.value())

    def __str__(self) -> str:
        def regex_object_prestringify(re_obj):
            return (re_obj.pattern, re_obj.flags) if re_obj else None

        syntax_mappings = []
        for syntax_mapping in self.syntax_mappings:
            syntax_mapping = copy.copy(syntax_mapping)

            syntax_mapping["first_line_match_compiled"] = [
                regex_object_prestringify(regex_object)
                for regex_object in syntax_mapping["first_line_match_compiled"]
            ]

            syntax_mappings.append(syntax_mapping)

        return json.dumps(syntax_mappings)

    def value(self, val=...):
        if val is ...:
            return self.syntax_mappings
        else:
            self.syntax_mappings = val

    def _find_syntax_file_paths(
        self, drop_duplicated: bool = True, remove_useless: bool = True
    ) -> list:
        """
        @brief find the path of all syntax files

        @param drop_duplicated if True, for a syntax, only the highest priority resource will be returned

        @return list[str] the path of all syntax files
        """

        syntax_files = []
        for syntax_file_ext in ST_SYNTAX_FILE_EXTS:
            syntax_files += sublime.find_resources("*" + syntax_file_ext)

        if drop_duplicated:
            # key   = syntax resource path without extension
            # value = the corresponding extension
            # example: { 'Packages/Java/Java': '.sublime-syntax' }
            syntax_griddle = {}

            for syntax_file in syntax_files:
                file_name, file_ext = os.path.splitext(syntax_file)

                # ".sublime-syntax" is always preferred than ".tmLanguage"
                if file_name not in syntax_griddle or file_ext == ".sublime-syntax":
                    syntax_griddle[file_name] = file_ext

            # combine a name and an extension back into a full path
            syntax_files = [name + ext for name, ext in syntax_griddle.items()]

        if remove_useless:
            useless_paths = ["/sublime_lib/tests/"]

            syntax_files = list(
                filter(
                    lambda syntax_file: all(
                        useless_path not in syntax_file for useless_path in useless_paths
                    ),
                    syntax_files,
                )
            )

        return sorted(syntax_files)

    def _build_syntax_mappings(self) -> list:
        return list(
            map(
                self._normalize_syntax_mapping_attrs,
                # prefer user settings than built-in rules
                self._build_syntax_mappings_from_user() + self._build_syntax_mappings_from_st(),
            )
        )

    def _build_syntax_mappings_from_user(self) -> list:
        """ load from user settings """

        mapping_settings = self.settings.get("syntax_mapping", {}).items()
        syntax_mappings = []

        for syntax_file_partial, first_line_matches in mapping_settings:
            # syntax_file_partial could be partial path
            # we try to get the full path here
            for syntax_file in self.syntax_files:
                if syntax_file_partial in syntax_file:
                    self.logger.info(
                        'Match syntax file "{}" with "{}"'.format(syntax_file_partial, syntax_file)
                    )

                    syntax_mappings.append({
                        "rule_source": "user",
                        "file_path": syntax_file,
                        "file_extensions": [],
                        "first_line_match": first_line_matches,
                    })

                    break
            else:
                self.logger.error('Cannot find a syntax file for "{}"'.format(syntax_file_partial))

        return syntax_mappings

    def _build_syntax_mappings_from_st(self) -> list:
        """ load from ST packages (one-time job, unless restart ST) """

        syntax_mappings = []

        for syntax_file in self.syntax_files:
            syntax_file_content = sublime.load_resource(syntax_file)

            attrs = self._get_attributes_from_syntax_file_content(
                syntax_file_content,
                [
                    # 'file_extensions' in YAML is the same usage with 'fileTypes' in XML
                    # but we perfer using 'file_extensions' because it's more self-explaining
                    "file_extensions",
                    "fileTypes",
                    "first_line_match",
                ],
            )

            if not attrs:
                self.logger.error("Failed to parse file: " + syntax_file)

                continue

            attrs.update({"rule_source": "ST", "file_path": syntax_file})

            syntax_mappings.append(attrs)

        return syntax_mappings

    def _normalize_syntax_mapping_attrs(self, attrs: dict) -> dict:
        attrs = copy.copy(attrs)

        # prevent from None and nonexisting keys
        if "file_types" not in attrs or not attrs["file_types"]:
            attrs["file_types"] = []
        if "file_extensions" not in attrs or not attrs["file_extensions"]:
            attrs["file_extensions"] = []
        if "first_line_match" not in attrs or not attrs["first_line_match"]:
            attrs["first_line_match"] = []

        if not isinstance(attrs["first_line_match"], list):
            attrs["first_line_match"] = [attrs["first_line_match"]]

        # compile "first_line_match" into "first_line_match_compiled"
        attrs["first_line_match_compiled"] = []
        for first_line_match in attrs["first_line_match"]:
            try:
                attrs["first_line_match_compiled"].append(
                    self._st_syntax_regex_compile(first_line_match)
                )
            except Exception as e:
                # we want to maintain the same index with "first_line_match"
                # so we apppend a None here
                attrs["first_line_match_compiled"].append(None)
                self.logger.error(
                    'Failed to compile "first_line_match" regex `{regex}` in "{syntax}" because {reason}'.format(
                        syntax=attrs["file_path"], regex=first_line_match, reason=e
                    )
                )

        # use 'file_extensions' as the formal key rather than 'file_types'
        attrs["file_extensions"] += attrs["file_types"]
        attrs["file_extensions"] = list(set(attrs["file_extensions"]))  # unique
        # remove the "file_types" key
        attrs.pop("file_types", None)

        return attrs

    def _st_syntax_regex_compile(self, regex: str):
        """
        @brief Compile regex which is from ST's syntax file
        @details Inline regex flag like "(?x: ... )" is unsupported in Python 3.3 (Python 3.6 is fine).

        @param self  The object
        @param regex The regular expression

        @return The compiled regex object
        """

        import sys

        # no atomic group in Python
        regex = re.sub(r"\(\?>", "(?:", regex)

        if sys.version_info >= (3, 6):
            return re.compile(regex)

        inline_flags = {
            "a": re.ASCII,
            "i": re.IGNORECASE,
            "L": re.LOCALE,
            "m": re.MULTILINE,
            "s": re.DOTALL,
            "u": re.UNICODE,  # default enabled in Python 3
            "x": re.VERBOSE,
        }

        re_pattern = r"\(\?(?P<flags>[{flags}]+):".format(flags="".join(inline_flags.keys()))

        re_flags = 0
        for m in re.finditer(re_pattern, regex):
            for flag in m.group("flags"):
                re_flags |= inline_flags[flag]

        if not (re_flags & re.ASCII):
            re_flags |= re.UNICODE

        regex = re.sub(re_pattern, "(?:", regex)

        return re.compile(regex, re_flags)

    def _get_attributes_from_syntax_file_content(self, content: str = "", attrs: list = []) -> dict:
        """ find "first_line_match" or "first_line_match" in syntax file content """

        attrs = map(camel_to_snake, attrs)

        if content.lstrip().startswith("<"):
            return self._get_attributes_from_xml_string(content, attrs)
        else:
            return self._get_attributes_from_yaml_string(content, attrs)

    def _get_attributes_from_yaml_string(self, content: str = "", attrs: list = []) -> dict:
        """ find attributes in .sublime-syntax content. "attrs" should be snake-cased. """

        try:
            # "contexts:" is usually the last (and largest) part of a syntax deinition.
            # to speed up searching, strip everything behinds "contexts:"
            cut_pos = content.find("contexts:")
            if cut_pos >= 0:
                content = content[:cut_pos]

            # prevent from failure because of parsing empty content
            content += "\n__workaround__: 1"

            parsed = yaml.safe_load(content)

            if parsed is None:
                raise Exception("failed to parse YAML content")
        except Exception:
            return None

        return {attr: parsed.get(attr, None) for attr in attrs}

    def _get_attributes_from_xml_string(self, content: str = "", attrs: list = []) -> dict:
        """ find attributes in .tmLanguage content. "attrs" should be snake-cased. """

        try:
            # "<key>patterns</key>" is usually the last (and largest) part of a syntax deinition.
            # to speed up searching, strip everything behinds "<key>patterns</key>"
            cut_pos = content.find("<key>patterns</key>")
            if cut_pos >= 0:
                content = content[:cut_pos] + "</dict></plist>"

            parsed = plistlib.readPlistFromBytes(content.encode("UTF-8"))
        except Exception:
            return None

        return {attr: parsed.get(snake_to_camel(attr), None) for attr in attrs}
<commit_msg>$ black .<commit_after>import copy
import json
import logging
import os
import plistlib
import re
import sublime
import yaml
from .functions import snake_to_camel, camel_to_snake

ST_SYNTAX_FILE_EXTS = [".sublime-syntax", ".tmLanguage"]

class SyntaxMappings(object):
    settings = None
    logger = None

    # items of this list are dicts whose keys are
    #     rule_source: str
    #     file_extensions: list[str]
    #     file_path: str
    #     first_line_match: list[str]
    #     first_line_match_compiled: list[compiled regex object]
    syntax_mappings = []

    # the path of all syntax files
    syntax_files = []

    def __init__(self, settings: sublime.Settings, logger: logging.Logger) -> None:
        self.settings = settings
        self.logger = logger

        self.syntax_files = self._find_syntax_file_paths()
        self.syntax_mappings = self._build_syntax_mappings()

        self.logger.debug("Found syntax files: {}".format(self.syntax_files))

    def __iter__(self):
        return iter(self.value())

    def __len__(self) -> int:
        return len(self.value())

    def __str__(self) -> str:
        def regex_object_prestringify(re_obj):
            return (re_obj.pattern, re_obj.flags) if re_obj else None

        syntax_mappings = []
        for syntax_mapping in self.syntax_mappings:
            syntax_mapping = copy.copy(syntax_mapping)

            syntax_mapping["first_line_match_compiled"] = [
                regex_object_prestringify(regex_object)
                for regex_object in syntax_mapping["first_line_match_compiled"]
            ]

            syntax_mappings.append(syntax_mapping)

        return json.dumps(syntax_mappings)

    def value(self, val=...):
        if val is ...:
            return self.syntax_mappings
        else:
            self.syntax_mappings = val

    def _find_syntax_file_paths(
        self, drop_duplicated: bool = True, remove_useless: bool = True
    ) -> list:
        """
        @brief find the path of all syntax files

        @param drop_duplicated if True, for a syntax, only the highest priority resource will be returned

        @return list[str] the path of all syntax files
        """

        syntax_files = []
        for syntax_file_ext in ST_SYNTAX_FILE_EXTS:
            syntax_files += sublime.find_resources("*" + syntax_file_ext)

        if drop_duplicated:
            # key   = syntax resource path without extension
            # value = the corresponding extension
            # example: { 'Packages/Java/Java': '.sublime-syntax' }
            syntax_griddle = {}

            for syntax_file in syntax_files:
                file_name, file_ext = os.path.splitext(syntax_file)

                # ".sublime-syntax" is always preferred than ".tmLanguage"
                if file_name not in syntax_griddle or file_ext == ".sublime-syntax":
                    syntax_griddle[file_name] = file_ext

            # combine a name and an extension back into a full path
            syntax_files = [name + ext for name, ext in syntax_griddle.items()]

        if remove_useless:
            useless_paths = ["/sublime_lib/tests/"]

            syntax_files = list(
                filter(
                    lambda syntax_file: all(
                        useless_path not in syntax_file for useless_path in useless_paths
                    ),
                    syntax_files,
                )
            )

        return sorted(syntax_files)

    def _build_syntax_mappings(self) -> list:
        return list(
            map(
                self._normalize_syntax_mapping_attrs,
                # prefer user settings than built-in rules
                self._build_syntax_mappings_from_user() + self._build_syntax_mappings_from_st(),
            )
        )

    def _build_syntax_mappings_from_user(self) -> list:
        """ load from user settings """

        mapping_settings = self.settings.get("syntax_mapping", {}).items()
        syntax_mappings = []

        for syntax_file_partial, first_line_matches in mapping_settings:
            # syntax_file_partial could be partial path
            # we try to get the full path here
            for syntax_file in self.syntax_files:
                if syntax_file_partial in syntax_file:
                    self.logger.info(
                        'Match syntax file "{}" with "{}"'.format(syntax_file_partial, syntax_file)
                    )

                    syntax_mappings.append(
                        {
                            "rule_source": "user",
                            "file_path": syntax_file,
                            "file_extensions": [],
                            "first_line_match": first_line_matches,
                        }
                    )

                    break
            else:
                self.logger.error('Cannot find a syntax file for "{}"'.format(syntax_file_partial))

        return syntax_mappings

    def _build_syntax_mappings_from_st(self) -> list:
        """ load from ST packages (one-time job, unless restart ST) """

        syntax_mappings = []

        for syntax_file in self.syntax_files:
            syntax_file_content = sublime.load_resource(syntax_file)

            attrs = self._get_attributes_from_syntax_file_content(
                syntax_file_content,
                [
                    # 'file_extensions' in YAML is the same usage with 'fileTypes' in XML
                    # but we perfer using 'file_extensions' because it's more self-explaining
                    "file_extensions",
                    "fileTypes",
                    "first_line_match",
                ],
            )

            if not attrs:
                self.logger.error("Failed to parse file: " + syntax_file)

                continue

            attrs.update({"rule_source": "ST", "file_path": syntax_file})

            syntax_mappings.append(attrs)

        return syntax_mappings

    def _normalize_syntax_mapping_attrs(self, attrs: dict) -> dict:
        attrs = copy.copy(attrs)

        # prevent from None and nonexisting keys
        if "file_types" not in attrs or not attrs["file_types"]:
            attrs["file_types"] = []
        if "file_extensions" not in attrs or not attrs["file_extensions"]:
            attrs["file_extensions"] = []
        if "first_line_match" not in attrs or not attrs["first_line_match"]:
            attrs["first_line_match"] = []

        if not isinstance(attrs["first_line_match"], list):
            attrs["first_line_match"] = [attrs["first_line_match"]]

        # compile "first_line_match" into "first_line_match_compiled"
        attrs["first_line_match_compiled"] = []
        for first_line_match in attrs["first_line_match"]:
            try:
                attrs["first_line_match_compiled"].append(
                    self._st_syntax_regex_compile(first_line_match)
                )
            except Exception as e:
                # we want to maintain the same index with "first_line_match"
                # so we apppend a None here
                attrs["first_line_match_compiled"].append(None)
                self.logger.error(
                    'Failed to compile "first_line_match" regex `{regex}` in "{syntax}" because {reason}'.format(
                        syntax=attrs["file_path"], regex=first_line_match, reason=e
                    )
                )

        # use 'file_extensions' as the formal key rather than 'file_types'
        attrs["file_extensions"] += attrs["file_types"]
        attrs["file_extensions"] = list(set(attrs["file_extensions"]))  # unique
        # remove the "file_types" key
        attrs.pop("file_types", None)

        return attrs

    def _st_syntax_regex_compile(self, regex: str):
        """
        @brief Compile regex which is from ST's syntax file
        @details Inline regex flag like "(?x: ... )" is unsupported in Python 3.3 (Python 3.6 is fine).

        @param self  The object
        @param regex The regular expression

        @return The compiled regex object
        """

        import sys

        # no atomic group in Python
        regex = re.sub(r"\(\?>", "(?:", regex)

        if sys.version_info >= (3, 6):
            return re.compile(regex)

        inline_flags = {
            "a": re.ASCII,
            "i": re.IGNORECASE,
            "L": re.LOCALE,
            "m": re.MULTILINE,
            "s": re.DOTALL,
            "u": re.UNICODE,  # default enabled in Python 3
            "x": re.VERBOSE,
        }

        re_pattern = r"\(\?(?P<flags>[{flags}]+):".format(flags="".join(inline_flags.keys()))

        re_flags = 0
        for m in re.finditer(re_pattern, regex):
            for flag in m.group("flags"):
                re_flags |= inline_flags[flag]

        if not (re_flags & re.ASCII):
            re_flags |= re.UNICODE

        regex = re.sub(re_pattern, "(?:", regex)

        return re.compile(regex, re_flags)

    def _get_attributes_from_syntax_file_content(self, content: str = "", attrs: list = []) -> dict:
        """ find "first_line_match" or "first_line_match" in syntax file content """

        attrs = map(camel_to_snake, attrs)

        if content.lstrip().startswith("<"):
            return self._get_attributes_from_xml_string(content, attrs)
        else:
            return self._get_attributes_from_yaml_string(content, attrs)

    def _get_attributes_from_yaml_string(self, content: str = "", attrs: list = []) -> dict:
        """ find attributes in .sublime-syntax content. "attrs" should be snake-cased. """

        try:
            # "contexts:" is usually the last (and largest) part of a syntax deinition.
            # to speed up searching, strip everything behinds "contexts:"
            cut_pos = content.find("contexts:")
            if cut_pos >= 0:
                content = content[:cut_pos]

            # prevent from failure because of parsing empty content
            content += "\n__workaround__: 1"

            parsed = yaml.safe_load(content)

            if parsed is None:
                raise Exception("failed to parse YAML content")
        except Exception:
            return None

        return {attr: parsed.get(attr, None) for attr in attrs}

    def _get_attributes_from_xml_string(self, content: str = "", attrs: list = []) -> dict:
        """ find attributes in .tmLanguage content. "attrs" should be snake-cased. """

        try:
            # "<key>patterns</key>" is usually the last (and largest) part of a syntax deinition.
            # to speed up searching, strip everything behinds "<key>patterns</key>"
            cut_pos = content.find("<key>patterns</key>")
            if cut_pos >= 0:
                content = content[:cut_pos] + "</dict></plist>"

            parsed = plistlib.readPlistFromBytes(content.encode("UTF-8"))
        except Exception:
            return None

        return {attr: parsed.get(snake_to_camel(attr), None) for attr in attrs}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Watchmaker yum worker."""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals, with_statement)

import io
import re

import six

import watchmaker.utils
from watchmaker.exceptions import WatchmakerError
from watchmaker.managers.platform import LinuxPlatformManager
from watchmaker.workers.base import WorkerBase

class Yum(WorkerBase, LinuxPlatformManager):
    """
    Install yum repos.

    Args:
        repo_map: (:obj:`list`)
            List of dictionaries containing a map of yum repo files to systems.
            (*Default*: ``[]``)

    """

    SUPPORTED_DISTS = ('amazon', 'centos', 'red hat')

    # Pattern used to match against the first line of /etc/system-release. A
    # match will contain two groups: the dist name (e.g. 'red hat' or 'amazon')
    # and the dist version (e.g. '6.8' or '2016.09').
    DIST_PATTERN = re.compile(
        r"^({0})"
        "(?:[^0-9]+)"
        r"([\d]|[\d]+[.][\d])"
        "(?:.*)"
        .format('|'.join(SUPPORTED_DISTS))
    )

    def __init__(self, *args, **kwargs):
        # Pop arguments used by Yum
        self.yumrepomap = kwargs.pop('repo_map', None) or []

        # Init inherited classes
        super(Yum, self).__init__(*args, **kwargs)
        self.dist_info = self.get_dist_info()

    def _get_amazon_el_version(self, version):
        # All amzn linux distros currently available use el6-based packages.
        # When/if amzn linux switches a distro to el7, rethink this.
        self.log.debug('Amazon Linux, version=%s', version)
        return '6'

    def get_dist_info(self):
        """Validate the Linux distro and return info about the distribution."""
        dist = None
        version = None
        el_version = None

        # Read first line from /etc/system-release
        try:
            with io.open('/etc/system-release', encoding='utf8') as fh_:
                release = fh_.readline().strip()
        except Exception:
            self.log.critical(
                'Failed to read /etc/system-release. Cannot determine system '
                'distribution!'
            )
            raise

        # Search the release file for a match against _supported_dists
        matched = self.DIST_PATTERN.search(release.lower())
        if matched is None:
            # Release not supported, exit with error
            msg = (
                'Unsupported OS distribution. OS must be one of: {0}'
                .format(', '.join(self.SUPPORTED_DISTS))
            )
            self.log.critical(msg)
            raise WatchmakerError(msg)

        # Assign dist,version from the match groups tuple, removing any spaces
        dist, version = (x.replace(' ', '') for x in matched.groups())

        # Determine el_version
        if dist == 'amazon':
            el_version = self._get_amazon_el_version(version)
        else:
            el_version = version.split('.')[0]

        if el_version is None:
            msg = (
                'Unsupported OS version! dist = {0}, version = {1}.'
                .format(dist, version)
            )
            self.log.critical(msg)
            raise WatchmakerError(msg)

        dist_info = {
            'dist': dist,
            'el_version': el_version
        }
        self.log.debug('dist_info=%s', dist_info)
        return dist_info

    def _validate_config(self):
        """Validate the config is properly formed."""
        if not self.yumrepomap:
            self.log.warning('`yumrepomap` did not exist or was empty.')
        elif not isinstance(self.yumrepomap, list):
            msg = '`yumrepomap` must be a list!'
            self.log.critical(msg)
            raise WatchmakerError(msg)

    def _validate_repo(self, repo):
        """Check if a repo is applicable to this system."""
        # Check if this repo applies to this system's dist and el_version.
        # repo['dist'] must match this system's dist or the keyword 'all'
        # repo['el_version'] is optional, but if present then it must match
        # this system's el_version.
        dist = self.dist_info['dist']
        el_version = self.dist_info['el_version']

        repo_dists = repo['dist']
        if isinstance(repo_dists, six.string_types):
            # ensure repo_dist is a list
            repo_dists = [repo_dists]

        # is repo dist applicable to this system?
        check_dist = bool(set(repo_dists).intersection([dist, 'all']))

        # is repo el_version applicable to this system?
        check_el_version = (
            'el_version' in repo and
            str(repo['el_version']) == str(el_version)
        )

        # return True if all checks pass, otherwise False
        return check_dist and check_el_version

    def before_install(self):
        """Validate configuration before starting install."""
        pass

    def install(self):
        """Install yum repos defined in config file."""
        self._validate_config()

        for repo in self.yumrepomap:
            if self._validate_repo(repo):
                # Download the yum repo definition to /etc/yum.repos.d/
                self.log.info('Installing repo: %s', repo['url'])
                url = repo['url']
                repofile = '/etc/yum.repos.d/{0}'.format(
                    watchmaker.utils.basename_from_uri(url))
                self.retrieve_file(url, repofile)
            else:
                self.log.debug(
                    'Skipped repo because it is not valid for this system: '
                    'dist_info=%s',
                    self.dist_info
                )
                self.log.debug('Skipped repo=%s', repo)
<commit_msg>Replace regex logic with distro module<commit_after># -*- coding: utf-8 -*-
"""Watchmaker yum worker."""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals, with_statement)

import io
import re

import distro
import six

import watchmaker.utils
from watchmaker.exceptions import WatchmakerError
from watchmaker.managers.platform import LinuxPlatformManager
from watchmaker.workers.base import WorkerBase

class Yum(WorkerBase, LinuxPlatformManager):
    """
    Install yum repos.

    Args:
        repo_map: (:obj:`list`)
            List of dictionaries containing a map of yum repo files to systems.
            (*Default*: ``[]``)

    """

    SUPPORTED_DISTS = ('amazon', 'centos', 'red hat', 'rhel')

    # Pattern used to match against the first line of /etc/system-release. A
    # match will contain two groups: the dist name (e.g. 'red hat' or 'amazon')
    # and the dist version (e.g. '6.8' or '2016.09').
    DIST_PATTERN = re.compile(
        r"^({0})"
        "(?:[^0-9]+)"
        r"([\d]|[\d]+[.][\d])"
        "(?:.*)"
        .format('|'.join(SUPPORTED_DISTS))
    )

    def __init__(self, *args, **kwargs):
        # Pop arguments used by Yum
        self.yumrepomap = kwargs.pop('repo_map', None) or []

        # Init inherited classes
        super(Yum, self).__init__(*args, **kwargs)
        self.dist_info = self.get_dist_info()

    def _get_amazon_el_version(self, version):
        # All amzn linux distros currently available use el6-based packages.
        # When/if amzn linux switches a distro to el7, rethink this.
        self.log.debug('Amazon Linux, version=%s', version)
        return '6'

    def get_dist_info(self):
        """Validate the Linux distro and return info about the distribution."""
        dist = dist.id()
        version = dist.version()[0]
        el_version = None

        # Determine el_version
        if dist == 'amazon':
            el_version = self._get_amazon_el_version(version)
        else:
            el_version = dist.version()[0]

        if el_version is None:
            msg = (
                'Unsupported OS version! dist = {0}, version = {1}.'
                .format(dist, version)
            )
            self.log.critical(msg)
            raise WatchmakerError(msg)

        dist_info = {
            'dist': dist,
            'el_version': el_version
        }
        self.log.debug('dist_info=%s', dist_info)
        return dist_info

    def _validate_config(self):
        """Validate the config is properly formed."""
        if not self.yumrepomap:
            self.log.warning('`yumrepomap` did not exist or was empty.')
        elif not isinstance(self.yumrepomap, list):
            msg = '`yumrepomap` must be a list!'
            self.log.critical(msg)
            raise WatchmakerError(msg)

    def _validate_repo(self, repo):
        """Check if a repo is applicable to this system."""
        # Check if this repo applies to this system's dist and el_version.
        # repo['dist'] must match this system's dist or the keyword 'all'
        # repo['el_version'] is optional, but if present then it must match
        # this system's el_version.
        dist = self.dist_info['dist']
        el_version = self.dist_info['el_version']

        repo_dists = repo['dist']
        if isinstance(repo_dists, six.string_types):
            # ensure repo_dist is a list
            repo_dists = [repo_dists]

        # is repo dist applicable to this system?
        check_dist = bool(set(repo_dists).intersection([dist, 'all']))

        # is repo el_version applicable to this system?
        check_el_version = (
            'el_version' in repo and
            str(repo['el_version']) == str(el_version)
        )

        # return True if all checks pass, otherwise False
        return check_dist and check_el_version

    def before_install(self):
        """Validate configuration before starting install."""
        pass

    def install(self):
        """Install yum repos defined in config file."""
        self._validate_config()

        for repo in self.yumrepomap:
            if self._validate_repo(repo):
                # Download the yum repo definition to /etc/yum.repos.d/
                self.log.info('Installing repo: %s', repo['url'])
                url = repo['url']
                repofile = '/etc/yum.repos.d/{0}'.format(
                    watchmaker.utils.basename_from_uri(url))
                self.retrieve_file(url, repofile)
            else:
                self.log.debug(
                    'Skipped repo because it is not valid for this system: '
                    'dist_info=%s',
                    self.dist_info
                )
                self.log.debug('Skipped repo=%s', repo)
<|endoftext|>


<commit_before>import pygame
import time
import datetime
import os
import Colors
import SwitchState
from photoslideshow import Photoslideshow

class Photobox():

    def __init__(self, windowsize, photofolder, camera, switch):
        pygame.init()
        self.windowsize = windowsize
        self.photofolder = photofolder
        self.screen = pygame.display.set_mode(
            self.windowsize, pygame.FULLSCREEN)
        self.clock = pygame.time.Clock()
        self.camera = camera
        self.switch = switch
        self.countdowntime = 3
        self.estimatedtriggertime = 2
        self.lastphototaken = datetime.datetime.now()
        pygame.mouse.set_visible(0)

    def start(self):
        slideshow = Photoslideshow(self)
        self.slideshow = slideshow
        while True:
            # don't crash the program if an error happens
            try:
                self.clearscreen()
                slideshow.shownextphoto()
                self.handleevents()
            except:
                pass
            self.exit_if_needed()

    def exit_if_needed(self):
        events = pygame.event.get()
        if len(events) == 0:
            return
        event = events[0]
        if event.type == pygame.KEYDOWN:
            key = event.key
            if key == pygame.K_ESCAPE:
                exit()

    def showphoto(self, path):
        self.clearscreen()
        photo = pygame.image.load(path)
        photo = pygame.transform.scale(photo, self.windowsize)
        self.screen.blit(photo, (0, 0))
        self.updatescreen()

    def updatescreen(self):
        pygame.display.flip()

    def clearscreen(self):
        self.screen.fill((0, 0, 0))

    def showtext(self, text, fontsize):
        self.clearscreen()
        font = pygame.font.Font(None, fontsize)
        fontWidth = font.size(text)[0]
        fontHeight = font.size(text)[1]
        midX = self.windowsize[0] / 2 - fontWidth / 2
        midY = self.windowsize[1] / 2 - fontHeight / 2
        screentext = font.render(text, True, Colors.randomcolor_rgb())
        self.screen.blit(screentext, (midX, midY))
        self.updatescreen()

    def handleevents(self):
        switchstate = self.switch.get_switch_state()
        print("%d Switchstate is %s" % (datetime.datetime.now().second, switchstate))
        if switchstate == SwitchState.SHUTDOWN:
            os.system("sudo shutdown now -h")
            exit(0)
        if switchstate == SwitchState.EXIT:
            exit(0)
        if self.newPhotoAllowed:
            if switchstate == SwitchState.TRIGGER:
                self.lastphototaken = datetime.datetime.now()
                self.takenewphoto()
                self.slideshow.reset_timer()

    def newPhotoAllowed(self):
        secondsdelta = self.countdowntime + self.estimatedtriggertime + 3
        nextphotoallowed = (self.lastphototaken +
                            datetime.timedelta(0, secondsdelta))
        timenow = datetime.datetime.now()
        if timenow > nextphotoallowed:
            return True
        return False

    def takenewphoto(self):
        self.showcountdown(self.countdowntime)
        self.showtext("Cheese!", 300)
        nextphotophath = self.photofolder.getnextfilename_fullpath()
        photoTaken = self.camera.takephoto(nextphotophath)
        if photoTaken:
            self.showphoto(nextphotophath)

    def showcountdown(self, upperbound):
        for i in range(upperbound, 0, -1):
            self.showtext(str(i), 400)
            time.sleep(1)
<commit_msg>Make nextphotoallowed a function call<commit_after>import pygame
import time
import datetime
import os
import Colors
import SwitchState
from photoslideshow import Photoslideshow

class Photobox():

    def __init__(self, windowsize, photofolder, camera, switch):
        pygame.init()
        self.windowsize = windowsize
        self.photofolder = photofolder
        self.screen = pygame.display.set_mode(
            self.windowsize, pygame.FULLSCREEN)
        self.clock = pygame.time.Clock()
        self.camera = camera
        self.switch = switch
        self.countdowntime = 3
        self.estimatedtriggertime = 2
        self.lastphototaken = datetime.datetime.now()
        pygame.mouse.set_visible(0)

    def start(self):
        slideshow = Photoslideshow(self)
        self.slideshow = slideshow
        while True:
            # don't crash the program if an error happens
            try:
                self.clearscreen()
                slideshow.shownextphoto()
                self.handleevents()
            except:
                pass
            self.exit_if_needed()

    def exit_if_needed(self):
        events = pygame.event.get()
        if len(events) == 0:
            return
        event = events[0]
        if event.type == pygame.KEYDOWN:
            key = event.key
            if key == pygame.K_ESCAPE:
                exit()

    def showphoto(self, path):
        self.clearscreen()
        photo = pygame.image.load(path)
        photo = pygame.transform.scale(photo, self.windowsize)
        self.screen.blit(photo, (0, 0))
        self.updatescreen()

    def updatescreen(self):
        pygame.display.flip()

    def clearscreen(self):
        self.screen.fill((0, 0, 0))

    def showtext(self, text, fontsize):
        self.clearscreen()
        font = pygame.font.Font(None, fontsize)
        fontWidth = font.size(text)[0]
        fontHeight = font.size(text)[1]
        midX = self.windowsize[0] / 2 - fontWidth / 2
        midY = self.windowsize[1] / 2 - fontHeight / 2
        screentext = font.render(text, True, Colors.randomcolor_rgb())
        self.screen.blit(screentext, (midX, midY))
        self.updatescreen()

    def handleevents(self):
        switchstate = self.switch.get_switch_state()
        if switchstate == SwitchState.SHUTDOWN:
            os.system("sudo shutdown now -h")
            exit(0)
        if switchstate == SwitchState.EXIT:
            exit(0)
        if self.newPhotoAllowed():
            if switchstate == SwitchState.TRIGGER:
                self.lastphototaken = datetime.datetime.now()
                self.takenewphoto()
                self.slideshow.reset_timer()

    def newPhotoAllowed(self):
        secondsdelta = self.countdowntime + self.estimatedtriggertime + 3
        nextphotoallowed = (self.lastphototaken +
                            datetime.timedelta(0, secondsdelta))
        timenow = datetime.datetime.now()
        if timenow > nextphotoallowed:
            return True
        return False

    def takenewphoto(self):
        self.showcountdown(self.countdowntime)
        self.showtext("Cheese!", 300)
        nextphotophath = self.photofolder.getnextfilename_fullpath()
        photoTaken = self.camera.takephoto(nextphotophath)
        if photoTaken:
            self.showphoto(nextphotophath)

    def showcountdown(self, upperbound):
        for i in range(upperbound, 0, -1):
            self.showtext(str(i), 400)
            time.sleep(1)
<|endoftext|>


<commit_before>import sys

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, String, Integer, Text, ForeignKey, desc,\
    create_engine, func
from sqlalchemy.dialects.mysql import DOUBLE, SMALLINT
from sqlalchemy.orm import sessionmaker, relationship
from sqlalchemy.exc import SQLAlchemyError

# Create the mappings
Base = declarative_base()

# The object to represent a record in the 'products' table
class Products(Base):
    __tablename__ = 'products'

    productCode = Column(String(15), primary_key=True)
    productName = Column(String(70))
    productLine = Column(String(50))
    productScale = Column(String(10))
    productVendor = Column(String(50))
    productDescription = Column(Text())
    quantityInStock = Column(SMALLINT(6))
    buyPrice = Column(DOUBLE())
    MSRP = Column(DOUBLE())

    def __repr__(self):
        return """<Product(productCode: {}, productName: {}, productLine: {},
        ProductScale: {}, productVendor: {}, productDescription: {},
        quantityInStock: {}, buyPrice: {}, MSRP: {}""".format(self.productCode,
                                                              self.productName,
                                                              self.productLine,
                                                              self.productScale,
                                                              self.productVendor,
                                                              self.productDescription,
                                                              self.quantityInStock,
                                                              self.buyPrice,
                                                              self.MSRP)

# The object to represent a record in the 'orderdetails' table
class OrderDetails(Base):
    __tablename__ = 'orderdetails'

    orderNumber = Column(Integer, primary_key=True)
    productCode = Column(String(15), ForeignKey('products.productCode'))
    quantityOrdered = Column(Integer)
    priceEach = Column(DOUBLE())
    orderLineNumber = Column(SMALLINT(6))

    products = relationship("Products")

    def __repr__(self):
        return """orderNumber: {}, productCode: {}, quantityOrdered: {},
        priceEach: {}, orderLineNumber: {}""".format(self.orderNumber,
                                                     self.productCode,
                                                     self.quantityOrdered,
                                                     self.priceEach,
                                                     self.orderLineNumber)

try:
    # Create an engine that connects us to the database
    engine = create_engine("mysql://root@localhost/classicmodels")

    # Create a session that acts as the intermediary between the database
    # and local changes
    Session = sessionmaker(bind=engine)
    session = Session()

    # Query the database for all the records in 'produts' and 'orderdetails'
    # and put them in respective query objects
    products = session.query(Products)
    orderdetails = session.query(OrderDetails)

    # This section is analagous to the ABL code:
    # FIND FIRST product where productVendor = Classic Metal Creations.
    query = products.filter(Products.productVendor == 'Classic Metal Creations')
    print query[0]

    # This section is analagous to the ABL code:
    # FIND LAST orderdetails where productCode = 'S18_3232'.
    index = -1
    print query[index]

    # This section is analagous to the ABL code:
    # FIND PREV orderdetails where productCode = 'S18_3232'
    index += -1
    print query[index]

    # This section (and the class MSRPCompare) is analagous to the
    # ABL code:
    # FOR EACH products WHERE products.quantityInStock > 5000
    #    BY products.MSRP DESCENDING:
    #    DISPLAY products.
    # END.

    for product in products.filter(Products.quantityInStock > 5000).\
        order_by(Products.MSRP.desc()):
        print product

    # This section is analogous to the ABL code:
    # FOR EACH orderdetails WHERE orderdetails.quantityOrdered >= 50,
    #     EACH products WHERE products.productCode = orderdetails.productCode:
    #     DISPLAY products.productName products.productCode orderdetails.quantityOrdered.
    # END.

    for o, p in orderdetails.outerjoin(Products, Products.productCode == OrderDetails.productCode)\
                            .with_entities(OrderDetails, Products).\
                            filter(OrderDetails.quantityOrdered >= 50):
        print p.productName, p.productCode, o.quantityOrdered

    # This section is analagous to the ABL code:
    # FOR EACH products:
    #     FOR EACH orderdetails WHERE orderdetails.productCode = products.productCode:
    #         ACCUMULATE orderdetails.orderNumber (COUNT).
    #         ACCUMULATE orderdetails.quantityOrdered (TOTAL).
    #     END.
    #     DISPLAY products.productName (ACCUM COUNT orderdetails.orderNumber)
    #         (ACCUM TOTAL orderdetails.quantityOrdered).
    # END.

    for productCode, numOrders, quantityOrdered in orderdetails.with_entities(
            OrderDetails.productCode,
            func.count(OrderDetails.productCode),
            func.sum(OrderDetails.quantityOrdered)).group_by(OrderDetails.productCode):
        print productCode, numOrders, quantityOrdered

except SQLAlchemyError, e:
    print e
    sys.exit(1)

finally:
    if session:
        session.close()
<commit_msg>Update orm.py<commit_after>import sys

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, String, Integer, Text, ForeignKey, desc,\
    create_engine, func
from sqlalchemy.dialects.mysql import DOUBLE, SMALLINT
from sqlalchemy.orm import sessionmaker, relationship
from sqlalchemy.exc import SQLAlchemyError

# Create the mappings
Base = declarative_base()

# The object to represent a record in the 'products' table
class Products(Base):
    __tablename__ = 'products'

    productCode = Column(String(15), primary_key=True)
    productName = Column(String(70))
    productLine = Column(String(50))
    productScale = Column(String(10))
    productVendor = Column(String(50))
    productDescription = Column(Text())
    quantityInStock = Column(SMALLINT(6))
    buyPrice = Column(DOUBLE())
    MSRP = Column(DOUBLE())

    def __repr__(self):
        return """<Product(productCode: {}, productName: {}, productLine: {},
        ProductScale: {}, productVendor: {}, productDescription: {},
        quantityInStock: {}, buyPrice: {}, MSRP: {}""".format(self.productCode,
                                                              self.productName,
                                                              self.productLine,
                                                              self.productScale,
                                                              self.productVendor,
                                                              self.productDescription,
                                                              self.quantityInStock,
                                                              self.buyPrice,
                                                              self.MSRP)

# The object to represent a record in the 'orderdetails' table
class OrderDetails(Base):
    __tablename__ = 'orderdetails'

    orderNumber = Column(Integer, primary_key=True)
    productCode = Column(String(15), ForeignKey('products.productCode'))
    quantityOrdered = Column(Integer)
    priceEach = Column(DOUBLE())
    orderLineNumber = Column(SMALLINT(6))

    products = relationship("Products")

    def __repr__(self):
        return """orderNumber: {}, productCode: {}, quantityOrdered: {},
        priceEach: {}, orderLineNumber: {}""".format(self.orderNumber,
                                                     self.productCode,
                                                     self.quantityOrdered,
                                                     self.priceEach,
                                                     self.orderLineNumber)

try:
    # Create an engine that connects us to the database
    engine = create_engine("mysql://root@localhost/classicmodels")

    # Create a session that acts as the intermediary between the database
    # and local changes
    Session = sessionmaker(bind=engine)
    session = Session()

    # Query the database for all the records in 'produts' and 'orderdetails'
    # and put them in respective query objects
    products = session.query(Products)
    orderdetails = session.query(OrderDetails)

    # This section is analagous to the ABL code:
    # FIND FIRST product where productVendor = Classic Metal Creations.
    query = products.filter(Products.productVendor == 'Classic Metal Creations')
    print query[0]

    # This section is analagous to the ABL code:
    # FIND LAST orderdetails where productCode = 'S18_3232'.
    index = -1
    print query[index]

    # This section is analagous to the ABL code:
    # FIND PREV orderdetails where productCode = 'S18_3232'
    index += -1
    print query[index]

    # This section (and the class MSRPCompare) is analagous to the
    # ABL code:
    # FOR EACH products WHERE products.quantityInStock > 5000
    #    BY products.MSRP DESCENDING:
    #    DISPLAY products.
    # END.

    for product in products.filter(Products.quantityInStock > 5000).\
        order_by(Products.MSRP.desc()):
        print product

    # This section is analogous to the ABL code:
    # FOR EACH orderdetails WHERE orderdetails.quantityOrdered >= 50,
    #     EACH products WHERE products.productCode = orderdetails.productCode:
    #     DISPLAY products.productName products.productCode orderdetails.quantityOrdered.
    # END.

    for o, p in orderdetails.join(Products, Products.productCode == OrderDetails.productCode)\
                            .with_entities(OrderDetails, Products).\
                            filter(OrderDetails.quantityOrdered >= 50):
        print p.productName, p.productCode, o.quantityOrdered

    # This section is analagous to the ABL code:
    # FOR EACH products:
    #     FOR EACH orderdetails WHERE orderdetails.productCode = products.productCode:
    #         ACCUMULATE orderdetails.orderNumber (COUNT).
    #         ACCUMULATE orderdetails.quantityOrdered (TOTAL).
    #     END.
    #     DISPLAY products.productName (ACCUM COUNT orderdetails.orderNumber)
    #         (ACCUM TOTAL orderdetails.quantityOrdered).
    # END.

    for productCode, numOrders, quantityOrdered in orderdetails.with_entities(
            OrderDetails.productCode,
            func.count(OrderDetails.productCode),
            func.sum(OrderDetails.quantityOrdered)).group_by(OrderDetails.productCode):
        print productCode, numOrders, quantityOrdered

except SQLAlchemyError, e:
    print e
    sys.exit(1)

finally:
    if session:
        session.close()
<|endoftext|>


<commit_before>import pytest
import numpy as np
import LabPy
from NonlinearTMM import TMM, Material

@pytest.fixture(params = [400e-9, 800e-9])
def wl(request):
    return request.param

@pytest.fixture(params = [1.5, 2.0])
def nPrism(request):
    return request.param

@pytest.fixture(params = [1.0, 1.3])
def nDielectric(request):
    return request.param

@pytest.fixture(params = ["p", "s"])
def pol(request):
    return request.param

@pytest.fixture(params = [50e-9, 100e-9])
def metalD(request):
    return request.param

@pytest.fixture(params = [0.1, 1.0])
def I0(request):
    return request.param

@pytest.fixture(params = [1.0, 1.0 + 3j, 1j])
def overrideE0(request):
    return request.param

@pytest.fixture(params = [0.0, 0.9, 1.4])
def betaTest(request):
    return request.param

@pytest.fixture()
def tmms1(request, wl, nPrism, nDielectric, pol, metalD, I0):
    prism = LabPy.Material("Static", n = nPrism)
    metal = LabPy.Material("main/Ag/Johnson")
    dielectric = LabPy.Material("Static", n = nDielectric)
    tmmCpp, tmmPy = GetTMMs(wl, pol, I0, prism, metal, dielectric, metalD, None)

    return tmmCpp, tmmPy

@pytest.fixture()
def tmms2(request, nPrism, nDielectric, pol, overrideE0):
    wl = 532e-9
    metalD = 50e-9
    I0 = 1.0

    prism = LabPy.Material("Static", n = nPrism)
    metal = LabPy.Material("main/Ag/Johnson")
    dielectric = LabPy.Material("Static", n = nDielectric)
    tmmCpp, tmmPy = GetTMMs(wl, pol, I0, prism, metal, dielectric, metalD, overrideE0)

    return tmmCpp, tmmPy

def test_Powerflows1(tmms1):
    TestPowerFlows(tmms1)

def test_Powerflows2(tmms2):
    TestPowerFlows(tmms2)

def test_Fields1D(tmms2, betaTest):
    tmmCpp, tmmPy = tmms2
    zs = np.linspace(-200e-9, 200e-9, 20)

    tmmCpp.Solve(beta = betaTest)
    tmmPy.Solve(beta = betaTest)

    resEPy, resHPy = tmmPy.GetFields(zs)
    resCpp = tmmCpp.GetFields(zs)

    np.testing.assert_allclose(resEPy, resCpp.E)
    np.testing.assert_allclose(resHPy, resCpp.H)

def test_Fields2D(tmms2, betaTest):
    tmmCpp, tmmPy = tmms2
    xs = np.linspace(-300e-9, 250e-9, 200)
    zs = np.linspace(-200e-9, 200e-9, 200)

    tmmCpp.Solve(beta = betaTest)
    tmmPy.Solve(beta = betaTest)

    resEPy, resHPy = tmmPy.GetFields2D(zs, xs)
    resCpp = tmmCpp.GetFields2D(zs, xs)

    if tmmPy.pol == "s":
        np.testing.assert_allclose(resEPy[:, :, 1], resCpp.Ey)
        np.testing.assert_allclose(resHPy[:, :, 0], resCpp.Hx)
        np.testing.assert_allclose(resHPy[:, :, 2], resCpp.Hz)
    else:
        np.testing.assert_allclose(resHPy[:, :, 1], resCpp.Hy)
        np.testing.assert_allclose(resEPy[:, :, 0], resCpp.Ex)
        np.testing.assert_allclose(resEPy[:, :, 2], resCpp.Ez)

#------------------------------------------------------------------------------

def GetTMMs(wl, pol, I0, prism, metal, dielectric, metalD, overrideE0):
    # C++ TMM
    tmmCpp = TMM()
    tmmCpp.SetParams(wl = wl, pol = pol, I0 = I0)

    if overrideE0 is not None:
        tmmCpp.SetParams(overrideE0 = True, E0 = overrideE0)

    tmmCpp.AddLayer(float("inf"), Material.FromLabPy(prism))
    tmmCpp.AddLayer(metalD, Material.FromLabPy(metal))
    tmmCpp.AddLayer(float("inf"), Material.FromLabPy(dielectric))

    # Python TMM
    tmmPy = LabPy._Tmm._NonlinearTmm._NonlinearTmm(wl = wl, pol = pol, I0 = I0, mode = "incident", overrideE0 = overrideE0) # @UndefinedVariable
    tmmPy.AddLayer(float("inf"), prism)
    tmmPy.AddLayer(metalD, metal)
    tmmPy.AddLayer(float("inf"), dielectric)
    tmmPy.SetParams(beta = 0.0)

    return tmmCpp, tmmPy

def TestPowerFlows(tmms):
    tmmCpp, tmmPy = tmms

    betas = np.linspace(0.0, 1.49, 20)
    sr = tmmCpp.Sweep("beta", betas)
    sweepFunc = np.vectorize(lambda beta: tmmPy.Solve(beta = beta))
    inc, r, t, I, R, T, _ = sweepFunc(betas)

    # Comparison
    np.testing.assert_allclose(sr.inc, inc)
    np.testing.assert_allclose(sr.r, r)
    np.testing.assert_allclose(sr.t, t)
    np.testing.assert_allclose(sr.Ii, I)
    np.testing.assert_allclose(sr.Ir, R)
    np.testing.assert_allclose(sr.It, T)

<commit_msg>Decrease rtol<commit_after>import pytest
import numpy as np
import LabPy
from NonlinearTMM import TMM, Material

@pytest.fixture(params = [400e-9, 800e-9])
def wl(request):
    return request.param

@pytest.fixture(params = [1.5, 2.0])
def nPrism(request):
    return request.param

@pytest.fixture(params = [1.0, 1.3])
def nDielectric(request):
    return request.param

@pytest.fixture(params = ["p", "s"])
def pol(request):
    return request.param

@pytest.fixture(params = [50e-9, 100e-9])
def metalD(request):
    return request.param

@pytest.fixture(params = [0.1, 1.0])
def I0(request):
    return request.param

@pytest.fixture(params = [1.0, 1.0 + 3j, 1j])
def overrideE0(request):
    return request.param

@pytest.fixture(params = [0.0, 0.9, 1.4])
def betaTest(request):
    return request.param

@pytest.fixture()
def tmms1(request, wl, nPrism, nDielectric, pol, metalD, I0):
    prism = LabPy.Material("Static", n = nPrism)
    metal = LabPy.Material("main/Ag/Johnson")
    dielectric = LabPy.Material("Static", n = nDielectric)
    tmmCpp, tmmPy = GetTMMs(wl, pol, I0, prism, metal, dielectric, metalD, None)

    return tmmCpp, tmmPy

@pytest.fixture()
def tmms2(request, nPrism, nDielectric, pol, overrideE0):
    wl = 532e-9
    metalD = 50e-9
    I0 = 1.0

    prism = LabPy.Material("Static", n = nPrism)
    metal = LabPy.Material("main/Ag/Johnson")
    dielectric = LabPy.Material("Static", n = nDielectric)
    tmmCpp, tmmPy = GetTMMs(wl, pol, I0, prism, metal, dielectric, metalD, overrideE0)

    return tmmCpp, tmmPy

def test_Powerflows1(tmms1):
    TestPowerFlows(tmms1)

def test_Powerflows2(tmms2):
    TestPowerFlows(tmms2)

def test_Fields1D(tmms2, betaTest):
    tmmCpp, tmmPy = tmms2
    zs = np.linspace(-200e-9, 200e-9, 20)

    tmmCpp.Solve(beta = betaTest)
    tmmPy.Solve(beta = betaTest)

    resEPy, resHPy = tmmPy.GetFields(zs)
    resCpp = tmmCpp.GetFields(zs)

    np.testing.assert_allclose(resEPy, resCpp.E)
    np.testing.assert_allclose(resHPy, resCpp.H)

def test_Fields2D(tmms2, betaTest):
    tmmCpp, tmmPy = tmms2
    xs = np.linspace(-300e-9, 250e-9, 200)
    zs = np.linspace(-200e-9, 200e-9, 200)

    tmmCpp.Solve(beta = betaTest)
    tmmPy.Solve(beta = betaTest)

    resEPy, resHPy = tmmPy.GetFields2D(zs, xs)
    resCpp = tmmCpp.GetFields2D(zs, xs)

    if tmmPy.pol == "s":
        np.testing.assert_allclose(resEPy[:, :, 1], resCpp.Ey)
        np.testing.assert_allclose(resHPy[:, :, 0], resCpp.Hx)
        np.testing.assert_allclose(resHPy[:, :, 2], resCpp.Hz)
    else:
        np.testing.assert_allclose(resHPy[:, :, 1], resCpp.Hy)
        np.testing.assert_allclose(resEPy[:, :, 0], resCpp.Ex)
        np.testing.assert_allclose(resEPy[:, :, 2], resCpp.Ez)

#------------------------------------------------------------------------------

def GetTMMs(wl, pol, I0, prism, metal, dielectric, metalD, overrideE0):
    # C++ TMM
    tmmCpp = TMM()
    tmmCpp.SetParams(wl = wl, pol = pol, I0 = I0)

    if overrideE0 is not None:
        tmmCpp.SetParams(overrideE0 = True, E0 = overrideE0)

    tmmCpp.AddLayer(float("inf"), Material.FromLabPy(prism))
    tmmCpp.AddLayer(metalD, Material.FromLabPy(metal))
    tmmCpp.AddLayer(float("inf"), Material.FromLabPy(dielectric))

    # Python TMM
    tmmPy = LabPy._Tmm._NonlinearTmm._NonlinearTmm(wl = wl, pol = pol, I0 = I0, mode = "incident", overrideE0 = overrideE0) # @UndefinedVariable
    tmmPy.AddLayer(float("inf"), prism)
    tmmPy.AddLayer(metalD, metal)
    tmmPy.AddLayer(float("inf"), dielectric)
    tmmPy.SetParams(beta = 0.0)

    return tmmCpp, tmmPy

def TestPowerFlows(tmms):
    tmmCpp, tmmPy = tmms

    betas = np.linspace(0.0, 1.49, 20)
    sr = tmmCpp.Sweep("beta", betas)
    sweepFunc = np.vectorize(lambda beta: tmmPy.Solve(beta = beta))
    inc, r, t, I, R, T, _ = sweepFunc(betas)

    # Comparison
    np.testing.assert_allclose(sr.inc, inc)
    np.testing.assert_allclose(sr.r, r)
    np.testing.assert_allclose(sr.t, t)
    np.testing.assert_allclose(sr.Ii, I)
    np.testing.assert_allclose(sr.Ir, R)
    np.testing.assert_allclose(sr.It, T, rtol = 1e-5)

<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['sina_download', 'sina_download_by_vid', 'sina_download_by_vkey']

from ..common import *

def video_info(id):
    xml = get_content('http://www.tucao.cc/api/sina.php?vid=%s' % id, decoded=True)
    urls = re.findall(r'<url>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</url>', xml)
    name = match1(xml, r'<vname>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vname>')
    vstr = match1(xml, r'<vstr>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vstr>')
    return urls, name, vstr

def sina_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vid.
    http://video.sina.com.cn/
    """

    urls, name, vstr = video_info(vid)
    title = title or name
    assert title
    size = 0
    for url in urls:
        _, _, temp = url_info(url)
        size += temp

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls(urls, title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download_by_vkey(vkey, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vkey.
    http://video.sina.com/
    """

    url = 'http://video.sina.com/v/flvideo/%s_0.flv' % vkey
    type, ext, size = url_info(url)

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download(url, output_dir='.', merge=True, info_only=False):
    """Downloads Sina videos by URL.
    """

    vid = match1(url, r'vid=(\d+)')
    if vid is None:
        video_page = get_content(url)
        vid = hd_vid = match1(video_page, r'hd_vid\s*:\s*\'([^\']+)\'')
        if hd_vid == '0':
            vids = match1(video_page, r'[^\w]vid\s*:\s*\'([^\']+)\'').split('|')
            vid = vids[-1]

    if vid:
        sina_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
    else:
        vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
        title = match1(video_page, r'title\s*:\s*"([^"]+)"')
        sina_download_by_vkey(vkey, title=title, output_dir=output_dir, merge=merge, info_only=info_only)

site_info = "Sina.com"
download = sina_download
download_playlist = playlist_not_supported('sina')
<commit_msg>Change sina API, fix #280<commit_after>#!/usr/bin/env python

__all__ = ['sina_download', 'sina_download_by_vid', 'sina_download_by_vkey']

from ..common import *

def video_info(id):
    xml = get_content('http://interface.bilibili.tv/playurl?vid=%s' % id, headers=fake_headers, decoded=True)
    #xml = get_content('http://www.tucao.cc/api/sina.php?vid=%s' % id, headers=fake_headers, decoded=True)
    urls = re.findall(r'<url>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</url>', xml)
    name = match1(xml, r'<vname>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vname>')
    vstr = match1(xml, r'<vstr>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vstr>')
    return urls, name, vstr

def sina_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vid.
    http://video.sina.com.cn/
    """

    urls, name, vstr = video_info(vid)
    title = title or name
    assert title
    size = 0
    for url in urls:
        _, _, temp = url_info(url)
        size += temp

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls(urls, title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download_by_vkey(vkey, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vkey.
    http://video.sina.com/
    """

    url = 'http://video.sina.com/v/flvideo/%s_0.flv' % vkey
    type, ext, size = url_info(url)

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)

def sina_download(url, output_dir='.', merge=True, info_only=False):
    """Downloads Sina videos by URL.
    """

    vid = match1(url, r'vid=(\d+)')
    if vid is None:
        video_page = get_content(url)
        vid = hd_vid = match1(video_page, r'hd_vid\s*:\s*\'([^\']+)\'')
        if hd_vid == '0':
            vids = match1(video_page, r'[^\w]vid\s*:\s*\'([^\']+)\'').split('|')
            vid = vids[-1]

    if vid:
        sina_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
    else:
        vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
        title = match1(video_page, r'title\s*:\s*"([^"]+)"')
        sina_download_by_vkey(vkey, title=title, output_dir=output_dir, merge=merge, info_only=info_only)

site_info = "Sina.com"
download = sina_download
download_playlist = playlist_not_supported('sina')
<|endoftext|>


<commit_before># Copyright (c) 2010-2011 gocept gmbh & co. kg
# See also LICENSE.txt

import datetime
import gocept.runner
import grokcore.component as grok
import itertools
import logging
import pytz
import zeit.cms.content.interfaces
import zeit.brightcove.converter
import zeit.cms.content.interfaces
import zeit.cms.repository.interfaces
import zeit.cms.workflow.interfaces
import zope.component

log = logging.getLogger(__name__)

@gocept.runner.transaction_per_item
def update_from_brightcove():
    # getting the playlists seems to be much more likely to fail, and since
    # we want to take what we can get, we do it *after* we have processed
    # the videos (instead of combining both steps)
    return itertools.chain(
        VideoUpdater.update_all(),
        PlaylistUpdater.update_all())

@gocept.runner.appmain(ticks=120, principal=gocept.runner.from_config(
    'zeit.brightcove', 'index-principal'))
def _update_from_brightcove():
    log.info('Update run started')
    update_from_brightcove()
    log.info('Update run finished')

class BaseUpdater(grok.Adapter):

    grok.baseclass()
    grok.implements(zeit.brightcove.interfaces.IUpdate)

    publish_priority = zeit.cms.workflow.interfaces.PRIORITY_DEFAULT

    def __init__(self, context):
        log.debug('%r(%s)', self, context.uniqueId)
        self.publish_job_id = None
        self.bcobj = context
        self.cmsobj = zeit.cms.interfaces.ICMSContent(
            self.bcobj.uniqueId, None)
        log.debug('CMS object resolved: %r', self.cmsobj)

    def __call__(self):
        success = self.delete() or self.add() or self.update()
        if not success:
            log.warning('Object %s not processed.', self.bcobj.uniqueId)

    @classmethod
    def repository(self):
        return zope.component.getUtility(
            zeit.cms.repository.interfaces.IRepository)

    @classmethod
    def update_all(cls):
        for x in cls.get_objects():
            cls.publish_priority = zeit.cms.workflow.interfaces.PRIORITY_LOW
            yield cls(x)

    @classmethod
    def get_objects(cls):
        raise NotImplementedError()

    def add(self):
        if self.cmsobj is None:
            log.info('Adding %s', self.bcobj)
            log.debug('Getting add location')
            folder = zeit.cms.content.interfaces.IAddLocation(self.bcobj)
            log.debug('Adding ...')
            folder[str(self.bcobj.id)] = self.bcobj.to_cms()
            self.cmsobj = folder[str(self.bcobj.id)]
            log.debug('Create publish job')
            self._publish_if_allowed()
            self.changed = True
            return True

    def delete(self):
        pass

    def update(self):
        pass

    def _update_cmsobj(self):
        log.info('Updating %s', self.bcobj)
        with zeit.cms.checkout.helper.checked_out(
            self.cmsobj, semantic_change=True, events=False) as co:
            # We don't need to send events here as a full checkout/checkin
            # cycle is done during publication anyway, below.
            if co is None:
                log.warning('Could not update %s' % self.cmsobj)
            else:
                self.bcobj.to_cms(co)
        self._publish_if_allowed()

    def _publish_cmsobj(self):
        info = zeit.cms.workflow.interfaces.IPublicationStatus(self.cmsobj)
        if info.published in ('not-published', 'published-with-changes'):
            self._publish_if_allowed()

    def _publish_if_allowed(self):
        self.publish_job_id = zeit.cms.workflow.interfaces.IPublish(
            self.cmsobj).publish(self.publish_priority)

class VideoUpdater(BaseUpdater):

    grok.context(zeit.brightcove.converter.Video)

    @classmethod
    def get_objects(cls):
        now = datetime.datetime.now(pytz.UTC)
        from_date = (datetime.datetime(now.year, now.month, now.day, now.hour)
                     - datetime.timedelta(hours=10))
        return zeit.brightcove.converter.Video.find_modified(
            from_date=from_date)

    def add(self):
        if not self.bcobj.ignore_for_update:
            return super(VideoUpdater, self).add()

    def delete(self):
        if self.bcobj.item_state == 'ACTIVE':
            # Handled elsewhere
            return False
        if self.bcobj.item_state == 'DELETED' and self.cmsobj is None:
            # Deleted in BC and no CMS object. We're done.
            return True
        if self.bcobj.item_state == 'INACTIVE' and self.cmsobj is None:
            # The item needs to be imported (but must not be published)
            return False
        if zeit.cms.workflow.interfaces.IPublishInfo(
                self.cmsobj).published:
            log.info('Retracting %s', self.bcobj)
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).retract(
                zeit.cms.workflow.interfaces.PRIORITY_LOW)
        elif self.bcobj.item_state == 'DELETED':
            log.info('Deleting %s', self.bcobj)
            if zeit.cms.workflow.interfaces.IPublishInfo(
                    self.cmsobj).published:
                zeit.cms.workflow.interfaces.IPublish(self.cmsobj).retract()
            folder = zeit.cms.content.interfaces.IAddLocation(self.bcobj)
            del folder[str(self.bcobj.id)]
            return True

    def update(self):
        # Update video in CMS iff the BC version is newer.
        new = self.bcobj.to_cms()
        changed = True

        # A bug in Brightcove may cause the last-modified date to remain
        # unchanged even when the video-still URL is actually changed.
        # (Also note that Brightcove has the misfeature of sometimes returning
        # old data for several minutes even directly after we POSTed to update
        # something.)
        if self.bcobj.ignore_for_update:
            return True
        current_mtime = zeit.cms.content.interfaces.ISemanticChange(
            self.cmsobj).last_semantic_change
        new_mtime = zeit.cms.content.interfaces.ISemanticChange(
            new).last_semantic_change
        if (current_mtime and new_mtime and current_mtime >= new_mtime and
            self.cmsobj.video_still == new.video_still):
            changed = False

        if changed:
            self._update_cmsobj()
        self._publish_cmsobj()
        self.changed = changed
        return True

    def _publish_if_allowed(self):
        if self.bcobj.item_state == 'ACTIVE':
            super(VideoUpdater, self)._publish_if_allowed()

class PlaylistUpdater(BaseUpdater):

    grok.context(zeit.brightcove.converter.Playlist)

    @classmethod
    def get_objects(cls):
        return zeit.brightcove.converter.Playlist.find_all()

    @classmethod
    def update_all(cls):
        objects = cls.get_objects()
        for x in objects:
            cls.publish_priority = zeit.cms.workflow.interfaces.PRIORITY_LOW
            yield cls(x)
        yield lambda: cls.delete_remaining_except(objects)

    def update(self):
        current = self.bcobj.from_cms(self.cmsobj)
        changed = False

        curdata = current.data
        newdata = self.bcobj.data
        for key in self.bcobj.fields.split(','):
            if key == 'id':
                continue
            if curdata.get(key) != newdata.get(key):
                changed = True
                break

        if changed:
            self._update_cmsobj()
        self._publish_cmsobj()
        self.changed = changed
        return True

    @classmethod
    def delete_remaining_except(cls, bc_objects):
        if not bc_objects:
            return
        folder = zeit.cms.content.interfaces.IAddLocation(
            iter(bc_objects).next())
        cms_names = set(folder.keys())
        bc_names = set(str(x.id) for x in bc_objects)
        for name in cms_names - bc_names:
            log.info('Deleting <Playlist id=%s>', name)
            cmsobj = folder[name]
            if zeit.cms.workflow.interfaces.IPublishInfo(cmsobj).published:
                zeit.cms.workflow.interfaces.IPublish(cmsobj).retract(
                    zeit.cms.workflow.interfaces.PRIORITY_LOW)
            else:
                del folder[name]
<commit_msg>use log functions string interpolation optimization<commit_after># Copyright (c) 2010-2011 gocept gmbh & co. kg
# See also LICENSE.txt

import datetime
import gocept.runner
import grokcore.component as grok
import itertools
import logging
import pytz
import zeit.cms.content.interfaces
import zeit.brightcove.converter
import zeit.cms.content.interfaces
import zeit.cms.repository.interfaces
import zeit.cms.workflow.interfaces
import zope.component

log = logging.getLogger(__name__)

@gocept.runner.transaction_per_item
def update_from_brightcove():
    # getting the playlists seems to be much more likely to fail, and since
    # we want to take what we can get, we do it *after* we have processed
    # the videos (instead of combining both steps)
    return itertools.chain(
        VideoUpdater.update_all(),
        PlaylistUpdater.update_all())

@gocept.runner.appmain(ticks=120, principal=gocept.runner.from_config(
    'zeit.brightcove', 'index-principal'))
def _update_from_brightcove():
    log.info('Update run started')
    update_from_brightcove()
    log.info('Update run finished')

class BaseUpdater(grok.Adapter):

    grok.baseclass()
    grok.implements(zeit.brightcove.interfaces.IUpdate)

    publish_priority = zeit.cms.workflow.interfaces.PRIORITY_DEFAULT

    def __init__(self, context):
        log.debug('%r(%s)', self, context.uniqueId)
        self.publish_job_id = None
        self.bcobj = context
        self.cmsobj = zeit.cms.interfaces.ICMSContent(
            self.bcobj.uniqueId, None)
        log.debug('CMS object resolved: %r', self.cmsobj)

    def __call__(self):
        success = self.delete() or self.add() or self.update()
        if not success:
            log.warning('Object %s not processed.', self.bcobj.uniqueId)

    @classmethod
    def repository(self):
        return zope.component.getUtility(
            zeit.cms.repository.interfaces.IRepository)

    @classmethod
    def update_all(cls):
        for x in cls.get_objects():
            cls.publish_priority = zeit.cms.workflow.interfaces.PRIORITY_LOW
            yield cls(x)

    @classmethod
    def get_objects(cls):
        raise NotImplementedError()

    def add(self):
        if self.cmsobj is None:
            log.info('Adding %s', self.bcobj)
            log.debug('Getting add location')
            folder = zeit.cms.content.interfaces.IAddLocation(self.bcobj)
            log.debug('Adding ...')
            folder[str(self.bcobj.id)] = self.bcobj.to_cms()
            self.cmsobj = folder[str(self.bcobj.id)]
            log.debug('Create publish job')
            self._publish_if_allowed()
            self.changed = True
            return True

    def delete(self):
        pass

    def update(self):
        pass

    def _update_cmsobj(self):
        log.info('Updating %s', self.bcobj)
        with zeit.cms.checkout.helper.checked_out(
            self.cmsobj, semantic_change=True, events=False) as co:
            # We don't need to send events here as a full checkout/checkin
            # cycle is done during publication anyway, below.
            if co is None:
                log.warning('Could not update %s', self.cmsobj)
            else:
                self.bcobj.to_cms(co)
        self._publish_if_allowed()

    def _publish_cmsobj(self):
        info = zeit.cms.workflow.interfaces.IPublicationStatus(self.cmsobj)
        if info.published in ('not-published', 'published-with-changes'):
            self._publish_if_allowed()

    def _publish_if_allowed(self):
        self.publish_job_id = zeit.cms.workflow.interfaces.IPublish(
            self.cmsobj).publish(self.publish_priority)

class VideoUpdater(BaseUpdater):

    grok.context(zeit.brightcove.converter.Video)

    @classmethod
    def get_objects(cls):
        now = datetime.datetime.now(pytz.UTC)
        from_date = (datetime.datetime(now.year, now.month, now.day, now.hour)
                     - datetime.timedelta(hours=10))
        return zeit.brightcove.converter.Video.find_modified(
            from_date=from_date)

    def add(self):
        if not self.bcobj.ignore_for_update:
            return super(VideoUpdater, self).add()

    def delete(self):
        if self.bcobj.item_state == 'ACTIVE':
            # Handled elsewhere
            return False
        if self.bcobj.item_state == 'DELETED' and self.cmsobj is None:
            # Deleted in BC and no CMS object. We're done.
            return True
        if self.bcobj.item_state == 'INACTIVE' and self.cmsobj is None:
            # The item needs to be imported (but must not be published)
            return False
        if zeit.cms.workflow.interfaces.IPublishInfo(
                self.cmsobj).published:
            log.info('Retracting %s', self.bcobj)
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).retract(
                zeit.cms.workflow.interfaces.PRIORITY_LOW)
        elif self.bcobj.item_state == 'DELETED':
            log.info('Deleting %s', self.bcobj)
            if zeit.cms.workflow.interfaces.IPublishInfo(
                    self.cmsobj).published:
                zeit.cms.workflow.interfaces.IPublish(self.cmsobj).retract()
            folder = zeit.cms.content.interfaces.IAddLocation(self.bcobj)
            del folder[str(self.bcobj.id)]
            return True

    def update(self):
        # Update video in CMS iff the BC version is newer.
        new = self.bcobj.to_cms()
        changed = True

        # A bug in Brightcove may cause the last-modified date to remain
        # unchanged even when the video-still URL is actually changed.
        # (Also note that Brightcove has the misfeature of sometimes returning
        # old data for several minutes even directly after we POSTed to update
        # something.)
        if self.bcobj.ignore_for_update:
            return True
        current_mtime = zeit.cms.content.interfaces.ISemanticChange(
            self.cmsobj).last_semantic_change
        new_mtime = zeit.cms.content.interfaces.ISemanticChange(
            new).last_semantic_change
        if (current_mtime and new_mtime and current_mtime >= new_mtime and
            self.cmsobj.video_still == new.video_still):
            changed = False

        if changed:
            self._update_cmsobj()
        self._publish_cmsobj()
        self.changed = changed
        return True

    def _publish_if_allowed(self):
        if self.bcobj.item_state == 'ACTIVE':
            super(VideoUpdater, self)._publish_if_allowed()

class PlaylistUpdater(BaseUpdater):

    grok.context(zeit.brightcove.converter.Playlist)

    @classmethod
    def get_objects(cls):
        return zeit.brightcove.converter.Playlist.find_all()

    @classmethod
    def update_all(cls):
        objects = cls.get_objects()
        for x in objects:
            cls.publish_priority = zeit.cms.workflow.interfaces.PRIORITY_LOW
            yield cls(x)
        yield lambda: cls.delete_remaining_except(objects)

    def update(self):
        current = self.bcobj.from_cms(self.cmsobj)
        changed = False

        curdata = current.data
        newdata = self.bcobj.data
        for key in self.bcobj.fields.split(','):
            if key == 'id':
                continue
            if curdata.get(key) != newdata.get(key):
                changed = True
                break

        if changed:
            self._update_cmsobj()
        self._publish_cmsobj()
        self.changed = changed
        return True

    @classmethod
    def delete_remaining_except(cls, bc_objects):
        if not bc_objects:
            return
        folder = zeit.cms.content.interfaces.IAddLocation(
            iter(bc_objects).next())
        cms_names = set(folder.keys())
        bc_names = set(str(x.id) for x in bc_objects)
        for name in cms_names - bc_names:
            log.info('Deleting <Playlist id=%s>', name)
            cmsobj = folder[name]
            if zeit.cms.workflow.interfaces.IPublishInfo(cmsobj).published:
                zeit.cms.workflow.interfaces.IPublish(cmsobj).retract(
                    zeit.cms.workflow.interfaces.PRIORITY_LOW)
            else:
                del folder[name]
<|endoftext|>