<commit_before>from __future__ import absolute_import

import logging
from datetime import date, timedelta
from dateutil.parser import parse

from dexter.app import celery_app as app
from dexter.processing import DocumentProcessor, DocumentProcessorNT

# force configs for API keys to be set
import dexter.core

# This is a collection of periodic tasks for Dexter, using
# Celery to drive task completion.

log = logging.getLogger(__name__)

@app.task
def back_process_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """

    if date.today() == date(2020, 8, 5):
        d1 = date(2020, 8, 3)
        days = [d1]

        filter_parm = ''

        for d in days:
            fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    # elif date.today() == date(2020, 6, 27):
    #     d1 = date(2020, 6, 24)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    # elif date.today() == date(2020, 6, 6):
    #     d1 = date(2020, 5, 30)
    #     # d2 = date(2020, 4, 21)
    #     # days = [d1 + timedelta(days=x) for x in range((d2 - d1).days + 1)]
    #     days = [d1]
    #
    # elif date.today() == date(2020, 6, 7):
    #     d1 = date(2020, 6, 1)
    #     # d2 = date(2020, 4, 21)
    #     # days = [d1 + timedelta(days=x) for x in range((d2 - d1).days + 1)]
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    else:
        print 'Already Done!'

@app.task
def fetch_yesterdays_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """
    yesterday = date.today() - timedelta(days=1)
    fetch_daily_feeds.delay(yesterday.isoformat())

# retry after 30 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=60, max_retries=7*24)
def fetch_filtered_daily_feeds(self, day, filter_parm):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_filtered_daily_feed_items(day, filter_parm):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry after 30 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=30*60, max_retries=7*24*2)
def fetch_daily_feeds(self, day):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_daily_feed_items(day):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry every minute, for up to 24 hours.
@app.task(bind=True, rate_limit="10/m", default_retry_delay=30, max_retries=2)
def get_feed_item(self, item):
    """ Fetch and process a document feed item. """
    try:
        dp = DocumentProcessorNT()
        dp.process_feed_item(item)
    except Exception as e:
        log.error("Error processing feed item: %s" % item, exc_info=e)
        self.retry()

@app.task
def backfill_taxonomies():
    """ Enqueue a task to backfill taxonomies """
    try:
        dp = DocumentProcessorNT()
        dp.backfill_taxonomies()
    except Exception as e:
        log.error("Error backfilling taxonomies: %s" % e.message, exc_info=e)
<commit_msg>Correct queue errors<commit_after>from __future__ import absolute_import

import logging
from datetime import date, timedelta
from dateutil.parser import parse

from dexter.app import celery_app as app
from dexter.processing import DocumentProcessor, DocumentProcessorNT

# force configs for API keys to be set
import dexter.core

# This is a collection of periodic tasks for Dexter, using
# Celery to drive task completion.

log = logging.getLogger(__name__)

@app.task
def back_process_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """

    if date.today() == date(2020, 8, 5):
        d1 = date(2020, 8, 3)
        days = [d1]

        filter_parm = ''

        for d in days:
            fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    # elif date.today() == date(2020, 6, 27):
    #     d1 = date(2020, 6, 24)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    # elif date.today() == date(2020, 6, 6):
    #     d1 = date(2020, 5, 30)
    #     # d2 = date(2020, 4, 21)
    #     # days = [d1 + timedelta(days=x) for x in range((d2 - d1).days + 1)]
    #     days = [d1]
    #
    # elif date.today() == date(2020, 6, 7):
    #     d1 = date(2020, 6, 1)
    #     # d2 = date(2020, 4, 21)
    #     # days = [d1 + timedelta(days=x) for x in range((d2 - d1).days + 1)]
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    else:
        print 'Already Done!'

@app.task
def fetch_yesterdays_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """
    yesterday = date.today() - timedelta(days=1)
    fetch_daily_feeds.delay(yesterday.isoformat())

# retry after 60 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=6*60*60, max_retries=7*4)
def fetch_filtered_daily_feeds(self, day, filter_parm):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_filtered_daily_feed_items(day, filter_parm):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry after 60 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=6*60*60, max_retries=7*4)
def fetch_daily_feeds(self, day):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_daily_feed_items(day):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry every minute, for up to 24 hours.
@app.task(bind=True, rate_limit="10/m", default_retry_delay=30, max_retries=2)
def get_feed_item(self, item):
    """ Fetch and process a document feed item. """
    try:
        dp = DocumentProcessorNT()
        dp.process_feed_item(item)
    except Exception as e:
        log.error("Error processing feed item: %s" % item, exc_info=e)
        self.retry()

@app.task
def backfill_taxonomies():
    """ Enqueue a task to backfill taxonomies """
    try:
        dp = DocumentProcessorNT()
        dp.backfill_taxonomies()
    except Exception as e:
        log.error("Error backfilling taxonomies: %s" % e.message, exc_info=e)
<|endoftext|>


<commit_before># coding: utf8

# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

"""
Project-independent library for Taskcluster decision tasks
"""

import datetime
import hashlib
import json
import os
import re
import sys
import taskcluster

class DecisionTask:
    DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"

    # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
    DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
        "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"

    def __init__(self, project_name, *, route_prefix,
                 worker_type="github-worker", docker_image_cache_expiry="1 year"):
        self.project_name = project_name
        self.route_prefix = route_prefix
        self.worker_type = worker_type
        self.docker_image_cache_expiry = docker_image_cache_expiry

        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
        self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
        self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})

        self.now = datetime.datetime.utcnow()
        self.built_images = {}

    def from_now_json(self, offset):
        return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))

    def find_or_build_image(self, dockerfile):
        image_build_task = self.built_images.get(dockerfile)
        if image_build_task is None:
            image_build_task = self._find_or_build_image(dockerfile)
            self.built_images[dockerfile] = image_build_task
        return image_build_task

    def _find_or_build_image(self, dockerfile):
        with open(dockerfile, "rb") as f:
            dockerfile_contents = f.read()
        digest = hashlib.sha256(dockerfile_contents).hexdigest()
        route = "%s.docker-image.%s" % (self.route_prefix, digest)

        try:
            result = self.index_service.findTask(route)
            return result["taskId"]
        except taskcluster.TaskclusterRestFailure as e:
            if e.status_code != 404:
                raise

        return self.create_task(
            task_name="docker image build task for image: " + self.image_name(dockerfile),
            script="""
                echo "$DOCKERFILE" | docker build -t taskcluster-built -
                docker save taskcluster-built | lz4 > /%s
            """ % self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            env={
                "DOCKERFILE": dockerfile_contents,
            },
            artifacts=[
                ("/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME, self.docker_image_cache_expiry),
            ],
            max_run_time_minutes=20,
            docker_image=self.DOCKER_IMAGE_BUILDER_IMAGE,
            features={
                "dind": True,  # docker-in-docker
            },
            with_repo=False,
            routes=[
                "index." + route,
            ],
            extra={
                "index": {
                    "expires": self.from_now_json(self.docker_image_cache_expiry),
                },
            },
        )

    def image_name(self, dockerfile):
        basename = os.path.basename(dockerfile)
        suffix = ".dockerfile"
        if basename == "Dockerfile":
            return os.path.basename(os.path.dirname(os.path.abspath(dockerfile)))
        elif basename.endswith(suffix):
            return basename[:-len(suffix)]
        else:
            return basename

    def create_task(self, *, task_name, script, max_run_time_minutes,
                    docker_image=None, dockerfile=None,  # One of these is required
                    artifacts=None, dependencies=None, env=None, cache=None, scopes=None,
                    routes=None, extra=None, features=None,
                    with_repo=True):
        if docker_image and dockerfile:
            raise TypeError("cannot use both `docker_image` or `dockerfile`")
        if not docker_image and not dockerfile:
            raise TypeError("need one of `docker_image` or `dockerfile`")

        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
        decision_task_id = os.environ["TASK_ID"]

        dependencies = [decision_task_id] + (dependencies or [])

        if dockerfile:
            image_build_task = self.find_or_build_image(dockerfile)
            dependencies.append(image_build_task)
            docker_image = {
                "type": "task-image",
                "taskId": image_build_task,
                "path": "public/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            }

        # Set in .taskcluster.yml
        task_owner = os.environ["TASK_OWNER"]
        task_source = os.environ["TASK_SOURCE"]

        env = env or {}

        if with_repo:
            # Set in .taskcluster.yml
            for k in ["GIT_URL", "GIT_REF", "GIT_SHA"]:
                env[k] = os.environ[k]

            script = """
                    git init repo
                    cd repo
                    git fetch --depth 1 "$GIT_URL" "$GIT_REF"
                    git reset --hard "$GIT_SHA"
                """ + script

        payload = {
            "taskGroupId": decision_task_id,
            "dependencies": dependencies or [],
            "schedulerId": "taskcluster-github",
            "provisionerId": "aws-provisioner-v1",
            "workerType": self.worker_type,

            "created": self.from_now_json(""),
            "deadline": self.from_now_json("1 day"),
            "metadata": {
                "name": "%s: %s" % (self.project_name, task_name),
                "description": "",
                "owner": task_owner,
                "source": task_source,
            },
            "scopes": scopes or [],
            "routes": routes or [],
            "extra": extra or {},
            "payload": {
                "cache": cache or {},
                "maxRunTime": max_run_time_minutes * 60,
                "image": docker_image,
                "command": [
                    "/bin/bash",
                    "--login",
                    "-x",
                    "-e",
                    "-c",
                    deindent(script)
                ],
                "env": env,
                "artifacts": {
                    "public/" + os.path.basename(path): {
                        "type": "file",
                        "path": path,
                        "expires": self.from_now_json(expires),
                    }
                    for path, expires in artifacts or []
                },
                "features": features or {},
            },
        }

        task_id = taskcluster.slugId().decode("utf8")
        self.queue_service.createTask(task_id, payload)
        print("Scheduled %s: %s" % (task_name, task_id))
        return task_id

def deindent(string):
    return re.sub("\n +", " \n ", string)
<commit_msg>Make image_name a function<commit_after># coding: utf8

# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

"""
Project-independent library for Taskcluster decision tasks
"""

import datetime
import hashlib
import json
import os
import re
import sys
import taskcluster

class DecisionTask:
    DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"

    # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
    DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
        "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"

    def __init__(self, project_name, *, route_prefix,
                 worker_type="github-worker", docker_image_cache_expiry="1 year"):
        self.project_name = project_name
        self.route_prefix = route_prefix
        self.worker_type = worker_type
        self.docker_image_cache_expiry = docker_image_cache_expiry

        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
        self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
        self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})

        self.now = datetime.datetime.utcnow()
        self.built_images = {}

    def from_now_json(self, offset):
        return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))

    def find_or_build_image(self, dockerfile):
        image_build_task = self.built_images.get(dockerfile)
        if image_build_task is None:
            image_build_task = self._find_or_build_image(dockerfile)
            self.built_images[dockerfile] = image_build_task
        return image_build_task

    def _find_or_build_image(self, dockerfile):
        with open(dockerfile, "rb") as f:
            dockerfile_contents = f.read()
        digest = hashlib.sha256(dockerfile_contents).hexdigest()
        route = "%s.docker-image.%s" % (self.route_prefix, digest)

        try:
            result = self.index_service.findTask(route)
            return result["taskId"]
        except taskcluster.TaskclusterRestFailure as e:
            if e.status_code != 404:
                raise

        return self.create_task(
            task_name="docker image build task for image: " + image_name(dockerfile),
            script="""
                echo "$DOCKERFILE" | docker build -t taskcluster-built -
                docker save taskcluster-built | lz4 > /%s
            """ % self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            env={
                "DOCKERFILE": dockerfile_contents,
            },
            artifacts=[
                ("/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME, self.docker_image_cache_expiry),
            ],
            max_run_time_minutes=20,
            docker_image=self.DOCKER_IMAGE_BUILDER_IMAGE,
            features={
                "dind": True,  # docker-in-docker
            },
            with_repo=False,
            routes=[
                "index." + route,
            ],
            extra={
                "index": {
                    "expires": self.from_now_json(self.docker_image_cache_expiry),
                },
            },
        )

    def create_task(self, *, task_name, script, max_run_time_minutes,
                    docker_image=None, dockerfile=None,  # One of these is required
                    artifacts=None, dependencies=None, env=None, cache=None, scopes=None,
                    routes=None, extra=None, features=None,
                    with_repo=True):
        if docker_image and dockerfile:
            raise TypeError("cannot use both `docker_image` or `dockerfile`")
        if not docker_image and not dockerfile:
            raise TypeError("need one of `docker_image` or `dockerfile`")

        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
        decision_task_id = os.environ["TASK_ID"]

        dependencies = [decision_task_id] + (dependencies or [])

        if dockerfile:
            image_build_task = self.find_or_build_image(dockerfile)
            dependencies.append(image_build_task)
            docker_image = {
                "type": "task-image",
                "taskId": image_build_task,
                "path": "public/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            }

        # Set in .taskcluster.yml
        task_owner = os.environ["TASK_OWNER"]
        task_source = os.environ["TASK_SOURCE"]

        env = env or {}

        if with_repo:
            # Set in .taskcluster.yml
            for k in ["GIT_URL", "GIT_REF", "GIT_SHA"]:
                env[k] = os.environ[k]

            script = """
                    git init repo
                    cd repo
                    git fetch --depth 1 "$GIT_URL" "$GIT_REF"
                    git reset --hard "$GIT_SHA"
                """ + script

        payload = {
            "taskGroupId": decision_task_id,
            "dependencies": dependencies or [],
            "schedulerId": "taskcluster-github",
            "provisionerId": "aws-provisioner-v1",
            "workerType": self.worker_type,

            "created": self.from_now_json(""),
            "deadline": self.from_now_json("1 day"),
            "metadata": {
                "name": "%s: %s" % (self.project_name, task_name),
                "description": "",
                "owner": task_owner,
                "source": task_source,
            },
            "scopes": scopes or [],
            "routes": routes or [],
            "extra": extra or {},
            "payload": {
                "cache": cache or {},
                "maxRunTime": max_run_time_minutes * 60,
                "image": docker_image,
                "command": [
                    "/bin/bash",
                    "--login",
                    "-x",
                    "-e",
                    "-c",
                    deindent(script)
                ],
                "env": env,
                "artifacts": {
                    "public/" + os.path.basename(path): {
                        "type": "file",
                        "path": path,
                        "expires": self.from_now_json(expires),
                    }
                    for path, expires in artifacts or []
                },
                "features": features or {},
            },
        }

        task_id = taskcluster.slugId().decode("utf8")
        self.queue_service.createTask(task_id, payload)
        print("Scheduled %s: %s" % (task_name, task_id))
        return task_id

def image_name(dockerfile):
    basename = os.path.basename(dockerfile)
    suffix = ".dockerfile"
    if basename == "Dockerfile":
        return os.path.basename(os.path.dirname(os.path.abspath(dockerfile)))
    elif basename.endswith(suffix):
        return basename[:-len(suffix)]
    else:
        return basename

def deindent(string):
    return re.sub("\n +", " \n ", string)
<|endoftext|>


<commit_before>from distutils.core import setup

with open('requirements.txt') as f:
    requirements = f.read().splitlines()

setup(name='DataHandling',
      version='1.5.0',
      description='Python package with functions for data analysis',
      author='Ashley Setter',
      author_email='A.Setter@soton.ac.uk',
      url=None,
      package_dir={'': 'DataHandling',
      },
      packages=['DataHandling', 'DataHandling.LeCroy', 'Datahandling.SimData'],
      install_requires=requirements,
)
<commit_msg>trying to fix travis pip installation<commit_after>from distutils.core import setup

with open('requirements.txt') as f:
    requirements = f.read().splitlines()

setup(name='DataHandling',
      version='1.5.0',
      description='Python package with functions for data analysis',
      author='Ashley Setter',
      author_email='A.Setter@soton.ac.uk',
      url=None,
      package_dir={'DataHandling': 'DataHandling',
                   'DataHandling.SimData': 'DataHandling/SimData',
      },
      packages=['DataHandling', 'DataHandling.LeCroy', 'Datahandling.SimData'],
      install_requires=requirements,
)
<|endoftext|>


<commit_before>from __future__ import print_function
from PySide import QtCore
from xml.etree import ElementTree
from collections import OrderedDict
import socket

try:
    from urllib2 import build_opener, URLError
    from urlparse import urljoin
except ImportError:  # >= 3.x
    from urllib.error import URLError
    from urllib.parse import urljoin
    from urllib.request import build_opener

class Modem(QtCore.QObject):
    """Modem - class that represents HiLink modem"""
    levelChanged = QtCore.Signal(int)
    statusChanged = QtCore.Signal(str, str)
    signalParamsChanged = QtCore.Signal(OrderedDict)
    unreadMessagesCountChanged = QtCore.Signal(int)
    finished = QtCore.Signal()

    def __init__(self, ip, interval):
        super(Modem, self).__init__()
        self._opener = build_opener()

        self._requestTimer = QtCore.QTimer()
        self._requestTimer.timeout.connect(self._updateInfo)

        self.ip = ip
        self.interval = interval

    @property
    def ip(self):
        return self._ip

    @ip.setter
    def ip(self, value):
        self._ip = value
        self._baseUrl = "http://{}".format(value)

    @property
    def interval(self):
        return self._interval

    @interval.setter
    def interval(self, value):
        self._interval = value
        # QTimer uses milliseconds
        self._requestTimer.setInterval(value * 1000)

    def finish(self):
        self.finished.emit()

    def connect_(self):
        msg = """<?xml version="1.0" encoding="UTF-8"?><request><dataswitch>1</dataswitch></request>"""

        self._post("/api/dialup/mobile-dataswitch", msg)

    def disconnect(self):
        msg = """<?xml version="1.0" encoding="UTF-8"?><request><dataswitch>0</dataswitch></request>"""

        self._post("/api/dialup/mobile-dataswitch", msg)

    def reboot(self):
        msg = """<?xml version="1.0" encoding="UTF-8"?><request><Control>1</Control></request>"""
        self._post("/api/device/control", msg)

    def _post(self, section, msg):
        response = self._opener.open(urljoin(self._baseUrl, section),
                                     data=msg.encode()).read()

    def _getXml(self, section):
        try:
            response = self._opener.open(urljoin(self._baseUrl, section),
                                         timeout=1).read()
        except (URLError, socket.timeout):
            return ElementTree.Element("")
        else:
            return ElementTree.fromstring(response)

    def _getTokens(self):
        """Get access tokens"""
        try:
            xml = self._getXml("/api/webserver/SesTokInfo")
        except (URLError, socket.timeout):
            return ("", "")
        else:
            return (xml.findtext("SesInfo", ""), xml.findtext("TokInfo", ""))

    def _updateTokens(self):
        session, postToken = self._getTokens()
        self._opener.addheaders = [("__RequestVerificationToken", postToken),
                                   ("Cookie", session)]

    def getSignalLevel(self, xml):
        return int(xml.findtext("SignalIcon") or "0")

    def getNetworkTypeEx(self, xml):
        types = {"0": "No service", "1": "GSM", "2": "GPRS", "3": "EDGE",
                 "21": "IS-95A", "22": "IS-95B", "23": "CDMA 1X",
                 "24": "EVDO Rev.0", "25": "EVDO Rev.A",
                 "26": "EVDO Rev.B", "27": "Hybrid CDMA 1X",
                 "28": "Hybrid EVDO Rev.0", "29": "Hybrid EVDO Rev.A",
                 "30": "Hybrid EVDO Rev.B", "31": "eHPRD Rel.0",
                 "32": "eHPRD Rel.A", "33": "eHPRD Rel.B",
                 "34": "Hybrid eHPRD Rel.0", "35": "Hybrid eHPRD Rel.A",
                 "36": "Hybrid eHPRD Rel.B", "41": "WCDMA", "42": "HSDPA",
                 "43": "HSUPA", "44": "HSPA", "45": "HSPA+", "46": "DC-HSPA+",
                 "61": "TD-SCDMA", "62": "TD-HSDPA", "63": "TD-HSUPA",
                 "64": "TD-HSPA", "65": "TD-HSPA+", "81": "802.16e",
                 "101": "LTE"}
        res = xml.findtext("CurrentNetworkTypeEx", "0")
        return types[res] if res != "" else None

    def getNetworkTypeCur(self, xml):
        types = {"0": "No service", "1": "GSM", "2": "GPRS", "3": "EDGE",
                 "4": "WCDMA", "5": "HSDPA", "6": "HSUPA", "7": "HSPA",
                 "8": "TD-SCDMA", "9": "HSPA+", "10": "EVDO Rev.0",
                 "11": "EVDO Rev.A", "12": "EVDO Rev.B",
                 "13": "1XRTT", "14": "UMB", "15": "1XEVDV",
                 "16": "3XRTT", "17": "HSPA+ 64QAM", "18": "HSPA+ MIMO"}
        res = xml.findtext("CurrentNetworkType")
        return types[res]

    def getNetworkType(self, xml):
        return self.getNetworkTypeEx(xml) or self.getNetworkTypeCur(xml)

    def getStatus(self, xml):
        states = {"900": "Connecting...", "901": "Connected",
                  "902": "Disconnected", "903": "Disconnecting..."}
        return states[xml.findtext("ConnectionStatus", "902")]

    def getOperator(self, xml):
        shortName = xml.find("ShortName")
        if shortName is not None:
            return shortName.text
        else:
            fullName = xml.find("FullName")
            if fullName is not None:
                return fullName.text
        return ""

    def getSignalParams(self, xml):
        params = ["rssi", "rsrp", "rsrq", "rscp", "ecio", "sinr",
                  "cell_id", "pci"]
        values = OrderedDict()

        for key in params:
            value = xml.findtext(key, "")
            if value:
                values[key] = "{key}: {val}".format(key=key.upper(),
                                                    val=value)
        return values

    def getUnreadMessageCount(self, xml):
        """Get number of unreaded messages"""
        unreadMessage = xml.find("UnreadMessage")
        if unreadMessage is not None:
            return int(unreadMessage.text)
        else:
            return 0

    def monitorMessages(self):
        """Monitor count of unreaded messages"""
        try:
            notifyXml = self._getXml("/api/monitoring/check-notifications")
            messageCount = self.getUnreadMessageCount(notifyXml)
        except URLError:
            self.unreadMessagesCountChanged.emit(0)
        else:
            self.unreadMessagesCountChanged.emit(messageCount)

    def monitorStatus(self):
        """Monitor connection status"""
        try:
            statusXml = self._getXml("/api/monitoring/status")
            plmnXml = self._getXml("/api/net/current-plmn")
        except (URLError, socket.timeout):
            self.levelChanged.emit(0)
            self.statusChanged.emit("Modem offline", "")
        else:
            signalLevel = self.getSignalLevel(statusXml)
            self.levelChanged.emit(signalLevel)

            status = self.getStatus(statusXml)
            networkType = self.getNetworkType(statusXml)
            operator = self.getOperator(plmnXml)

            # fix space if we don't have op name
            op = networkType
            if operator:
                op = "%s %s" % (operator, networkType)
            self.statusChanged.emit(status, op)

    def monitorSignalParams(self):
        """Monitor signal parameters"""
        try:
            signalXml = self._getXml("/api/device/signal")
        except (URLError, socket.timeout):
            self.signalParamsChanged.emit({})
        else:
            params = self.getSignalParams(signalXml)
            self.signalParamsChanged.emit(params)

    def monitor(self):
        self._requestTimer.start()

    def _updateInfo(self):
        self._updateTokens()
        self.monitorMessages()
        self.monitorStatus()
        self.monitorSignalParams()
<commit_msg>set modem as timer parent to move to other thread<commit_after>from __future__ import print_function
from PySide import QtCore
from xml.etree import ElementTree
from collections import OrderedDict
import socket

try:
    from urllib2 import build_opener, URLError
    from urlparse import urljoin
except ImportError:  # >= 3.x
    from urllib.error import URLError
    from urllib.parse import urljoin
    from urllib.request import build_opener

class Modem(QtCore.QObject):
    """Modem - class that represents HiLink modem"""
    levelChanged = QtCore.Signal(int)
    statusChanged = QtCore.Signal(str, str)
    signalParamsChanged = QtCore.Signal(OrderedDict)
    unreadMessagesCountChanged = QtCore.Signal(int)
    finished = QtCore.Signal()

    def __init__(self, ip, interval):
        super(Modem, self).__init__()
        self._opener = build_opener()

        self._requestTimer = QtCore.QTimer(self)
        self._requestTimer.timeout.connect(self._updateInfo)

        self.ip = ip
        self.interval = interval

    @property
    def ip(self):
        return self._ip

    @ip.setter
    def ip(self, value):
        self._ip = value
        self._baseUrl = "http://{}".format(value)

    @property
    def interval(self):
        return self._interval

    @interval.setter
    def interval(self, value):
        self._interval = value
        # QTimer uses milliseconds
        self._requestTimer.setInterval(value * 1000)

    def finish(self):
        self.finished.emit()

    def connect_(self):
        msg = """<?xml version="1.0" encoding="UTF-8"?><request><dataswitch>1</dataswitch></request>"""

        self._post("/api/dialup/mobile-dataswitch", msg)

    def disconnect(self):
        msg = """<?xml version="1.0" encoding="UTF-8"?><request><dataswitch>0</dataswitch></request>"""

        self._post("/api/dialup/mobile-dataswitch", msg)

    def reboot(self):
        msg = """<?xml version="1.0" encoding="UTF-8"?><request><Control>1</Control></request>"""
        self._post("/api/device/control", msg)

    def _post(self, section, msg):
        response = self._opener.open(urljoin(self._baseUrl, section),
                                     data=msg.encode()).read()

    def _getXml(self, section):
        try:
            response = self._opener.open(urljoin(self._baseUrl, section),
                                         timeout=1).read()
        except (URLError, socket.timeout):
            return ElementTree.Element("")
        else:
            return ElementTree.fromstring(response)

    def _getTokens(self):
        """Get access tokens"""
        try:
            xml = self._getXml("/api/webserver/SesTokInfo")
        except (URLError, socket.timeout):
            return ("", "")
        else:
            return (xml.findtext("SesInfo", ""), xml.findtext("TokInfo", ""))

    def _updateTokens(self):
        session, postToken = self._getTokens()
        self._opener.addheaders = [("__RequestVerificationToken", postToken),
                                   ("Cookie", session)]

    def getSignalLevel(self, xml):
        return int(xml.findtext("SignalIcon") or "0")

    def getNetworkTypeEx(self, xml):
        types = {"0": "No service", "1": "GSM", "2": "GPRS", "3": "EDGE",
                 "21": "IS-95A", "22": "IS-95B", "23": "CDMA 1X",
                 "24": "EVDO Rev.0", "25": "EVDO Rev.A",
                 "26": "EVDO Rev.B", "27": "Hybrid CDMA 1X",
                 "28": "Hybrid EVDO Rev.0", "29": "Hybrid EVDO Rev.A",
                 "30": "Hybrid EVDO Rev.B", "31": "eHPRD Rel.0",
                 "32": "eHPRD Rel.A", "33": "eHPRD Rel.B",
                 "34": "Hybrid eHPRD Rel.0", "35": "Hybrid eHPRD Rel.A",
                 "36": "Hybrid eHPRD Rel.B", "41": "WCDMA", "42": "HSDPA",
                 "43": "HSUPA", "44": "HSPA", "45": "HSPA+", "46": "DC-HSPA+",
                 "61": "TD-SCDMA", "62": "TD-HSDPA", "63": "TD-HSUPA",
                 "64": "TD-HSPA", "65": "TD-HSPA+", "81": "802.16e",
                 "101": "LTE"}
        res = xml.findtext("CurrentNetworkTypeEx", "0")
        return types[res] if res != "" else None

    def getNetworkTypeCur(self, xml):
        types = {"0": "No service", "1": "GSM", "2": "GPRS", "3": "EDGE",
                 "4": "WCDMA", "5": "HSDPA", "6": "HSUPA", "7": "HSPA",
                 "8": "TD-SCDMA", "9": "HSPA+", "10": "EVDO Rev.0",
                 "11": "EVDO Rev.A", "12": "EVDO Rev.B",
                 "13": "1XRTT", "14": "UMB", "15": "1XEVDV",
                 "16": "3XRTT", "17": "HSPA+ 64QAM", "18": "HSPA+ MIMO"}
        res = xml.findtext("CurrentNetworkType")
        return types[res]

    def getNetworkType(self, xml):
        return self.getNetworkTypeEx(xml) or self.getNetworkTypeCur(xml)

    def getStatus(self, xml):
        states = {"900": "Connecting...", "901": "Connected",
                  "902": "Disconnected", "903": "Disconnecting..."}
        return states[xml.findtext("ConnectionStatus", "902")]

    def getOperator(self, xml):
        shortName = xml.find("ShortName")
        if shortName is not None:
            return shortName.text
        else:
            fullName = xml.find("FullName")
            if fullName is not None:
                return fullName.text
        return ""

    def getSignalParams(self, xml):
        params = ["rssi", "rsrp", "rsrq", "rscp", "ecio", "sinr",
                  "cell_id", "pci"]
        values = OrderedDict()

        for key in params:
            value = xml.findtext(key, "")
            if value:
                values[key] = "{key}: {val}".format(key=key.upper(),
                                                    val=value)
        return values

    def getUnreadMessageCount(self, xml):
        """Get number of unreaded messages"""
        unreadMessage = xml.find("UnreadMessage")
        if unreadMessage is not None:
            return int(unreadMessage.text)
        else:
            return 0

    def monitorMessages(self):
        """Monitor count of unreaded messages"""
        try:
            notifyXml = self._getXml("/api/monitoring/check-notifications")
            messageCount = self.getUnreadMessageCount(notifyXml)
        except URLError:
            self.unreadMessagesCountChanged.emit(0)
        else:
            self.unreadMessagesCountChanged.emit(messageCount)

    def monitorStatus(self):
        """Monitor connection status"""
        try:
            statusXml = self._getXml("/api/monitoring/status")
            plmnXml = self._getXml("/api/net/current-plmn")
        except (URLError, socket.timeout):
            self.levelChanged.emit(0)
            self.statusChanged.emit("Modem offline", "")
        else:
            signalLevel = self.getSignalLevel(statusXml)
            self.levelChanged.emit(signalLevel)

            status = self.getStatus(statusXml)
            networkType = self.getNetworkType(statusXml)
            operator = self.getOperator(plmnXml)

            # fix space if we don't have op name
            op = networkType
            if operator:
                op = "%s %s" % (operator, networkType)
            self.statusChanged.emit(status, op)

    def monitorSignalParams(self):
        """Monitor signal parameters"""
        try:
            signalXml = self._getXml("/api/device/signal")
        except (URLError, socket.timeout):
            self.signalParamsChanged.emit({})
        else:
            params = self.getSignalParams(signalXml)
            self.signalParamsChanged.emit(params)

    def monitor(self):
        self._requestTimer.start()

    def _updateInfo(self):
        self._updateTokens()
        self.monitorMessages()
        self.monitorStatus()
        self.monitorSignalParams()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

class AnsibleCloudStack:

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.result = {
            'changed': False,
        }

        self.module = module
        self._connect()

        self.domain = None
        self.account = None
        self.project = None
        self.ip_address = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None
        self.capabilities = None

    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('secret_key')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')
        api_timeout = self.module.params.get('api_timeout')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                timeout=api_timeout,
                method=api_http_method
                )
        else:
            self.cs = CloudStack(**read_config())

    def get_or_fallback(self, key=None, fallback_key=None):
        value = self.module.params.get(key)
        if not value:
            value = self.module.params.get(fallback_key)
        return value

    # TODO: for backward compatibility only, remove if not used anymore
    def _has_changed(self, want_dict, current_dict, only_keys=None):
        return self.has_changed(want_dict=want_dict, current_dict=current_dict, only_keys=only_keys)

    def has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue;

            # Skip None values
            if value is None:
                continue;

            if key in current_dict:

                # API returns string for int in some cases, just to make sure
                if isinstance(value, int):
                    current_dict[key] = int(current_dict[key])
                elif isinstance(value, str):
                    current_dict[key] = str(current_dict[key])

                # Only need to detect a singe change, not every item
                if value != current_dict[key]:
                    return True
        return False

    def _get_by_key(self, key=None, my_dict={}):
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict

    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None
        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        projects = self.cs.listProjects(**args)
        if projects:
            for p in projects['project']:
                if project.lower() in [ p['name'].lower(), p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)

    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)

    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        args['zoneid'] = self.get_zone(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm in [ v['name'], v['displayname'], v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)

    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone in [ z['name'], z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)

    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)

    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)

    def get_account(self, key=None):
        if self.account:
            return self._get_by_key(key, self.account)

        account = self.module.params.get('account')
        if not account:
            return None

        domain = self.module.params.get('domain')
        if not domain:
            self.module.fail_json(msg="Account must be specified with Domain")

        args = {}
        args['name'] = account
        args['domainid'] = self.get_domain(key='id')
        args['listall'] = True
        accounts = self.cs.listAccounts(**args)
        if accounts:
            self.account = accounts['account'][0]
            return self._get_by_key(key, self.account)
        self.module.fail_json(msg="Account '%s' not found" % account)

    def get_domain(self, key=None):
        if self.domain:
            return self._get_by_key(key, self.domain)

        domain = self.module.params.get('domain')
        if not domain:
            return None

        args = {}
        args['listall'] = True
        domains = self.cs.listDomains(**args)
        if domains:
            for d in domains['domain']:
                if d['path'].lower() in [ domain.lower(), "root/" + domain.lower(), "root" + domain.lower() ]:
                    self.domain = d
                    return self._get_by_key(key, self.domain)
        self.module.fail_json(msg="Domain '%s' not found" % domain)

    def get_tags(self, resource=None):
        existing_tags = self.cs.listTags(resourceid=resource['id'])
        if existing_tags:
            return existing_tags['tag']
        return []

    def _delete_tags(self, resource, resource_type, tags):
        existing_tags = resource['tags']
        tags_to_delete = []
        for existing_tag in existing_tags:
            if existing_tag['key'] in tags:
                if existing_tag['value'] != tags[key]:
                    tags_to_delete.append(existing_tag)
            else:
                tags_to_delete.append(existing_tag)
        if tags_to_delete:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags_to_delete
                self.cs.deleteTags(**args)

    def _create_tags(self, resource, resource_type, tags):
        tags_to_create = []
        for i, tag_entry in enumerate(tags):
            tag = {
                'key':   tag_entry['key'],
                'value': tag_entry['value'],
            }
            tags_to_create.append(tag)
        if tags_to_create:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags_to_create
                self.cs.createTags(**args)

    def ensure_tags(self, resource, resource_type=None):
        if not resource_type or not resource:
            self.module.fail_json(msg="Error: Missing resource or resource_type for tags.")

        if 'tags' in resource:
            tags = self.module.params.get('tags')
            if tags is not None:
                self._delete_tags(resource, resource_type, tags)
                self._create_tags(resource, resource_type, tags)
                resource['tags'] = self.get_tags(resource)
        return resource

    def get_capabilities(self, key=None):
        if self.capabilities:
            return self._get_by_key(key, self.capabilities)
        capabilities = self.cs.listCapabilities()
        self.capabilities = capabilities['capability']
        return self._get_by_key(key, self.capabilities)

    # TODO: for backward compatibility only, remove if not used anymore
    def _poll_job(self, job=None, key=None):
        return self.poll_job(job=job, key=key)

    def poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job
<commit_msg>cloudstack: add get_result() in utils to return common results<commit_after># -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

class AnsibleCloudStack:

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.result = {
            'changed': False,
        }

        self.module = module
        self._connect()

        self.domain = None
        self.account = None
        self.project = None
        self.ip_address = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None
        self.capabilities = None

    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('secret_key')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')
        api_timeout = self.module.params.get('api_timeout')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                timeout=api_timeout,
                method=api_http_method
                )
        else:
            self.cs = CloudStack(**read_config())

    def get_or_fallback(self, key=None, fallback_key=None):
        value = self.module.params.get(key)
        if not value:
            value = self.module.params.get(fallback_key)
        return value

    # TODO: for backward compatibility only, remove if not used anymore
    def _has_changed(self, want_dict, current_dict, only_keys=None):
        return self.has_changed(want_dict=want_dict, current_dict=current_dict, only_keys=only_keys)

    def has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue;

            # Skip None values
            if value is None:
                continue;

            if key in current_dict:

                # API returns string for int in some cases, just to make sure
                if isinstance(value, int):
                    current_dict[key] = int(current_dict[key])
                elif isinstance(value, str):
                    current_dict[key] = str(current_dict[key])

                # Only need to detect a singe change, not every item
                if value != current_dict[key]:
                    return True
        return False

    def _get_by_key(self, key=None, my_dict={}):
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict

    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None
        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        projects = self.cs.listProjects(**args)
        if projects:
            for p in projects['project']:
                if project.lower() in [ p['name'].lower(), p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)

    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)

    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        args['zoneid'] = self.get_zone(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm in [ v['name'], v['displayname'], v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)

    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone in [ z['name'], z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)

    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)

    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)

    def get_account(self, key=None):
        if self.account:
            return self._get_by_key(key, self.account)

        account = self.module.params.get('account')
        if not account:
            return None

        domain = self.module.params.get('domain')
        if not domain:
            self.module.fail_json(msg="Account must be specified with Domain")

        args = {}
        args['name'] = account
        args['domainid'] = self.get_domain(key='id')
        args['listall'] = True
        accounts = self.cs.listAccounts(**args)
        if accounts:
            self.account = accounts['account'][0]
            return self._get_by_key(key, self.account)
        self.module.fail_json(msg="Account '%s' not found" % account)

    def get_domain(self, key=None):
        if self.domain:
            return self._get_by_key(key, self.domain)

        domain = self.module.params.get('domain')
        if not domain:
            return None

        args = {}
        args['listall'] = True
        domains = self.cs.listDomains(**args)
        if domains:
            for d in domains['domain']:
                if d['path'].lower() in [ domain.lower(), "root/" + domain.lower(), "root" + domain.lower() ]:
                    self.domain = d
                    return self._get_by_key(key, self.domain)
        self.module.fail_json(msg="Domain '%s' not found" % domain)

    def get_tags(self, resource=None):
        existing_tags = self.cs.listTags(resourceid=resource['id'])
        if existing_tags:
            return existing_tags['tag']
        return []

    def _delete_tags(self, resource, resource_type, tags):
        existing_tags = resource['tags']
        tags_to_delete = []
        for existing_tag in existing_tags:
            if existing_tag['key'] in tags:
                if existing_tag['value'] != tags[key]:
                    tags_to_delete.append(existing_tag)
            else:
                tags_to_delete.append(existing_tag)
        if tags_to_delete:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags_to_delete
                self.cs.deleteTags(**args)

    def _create_tags(self, resource, resource_type, tags):
        tags_to_create = []
        for i, tag_entry in enumerate(tags):
            tag = {
                'key':   tag_entry['key'],
                'value': tag_entry['value'],
            }
            tags_to_create.append(tag)
        if tags_to_create:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags_to_create
                self.cs.createTags(**args)

    def ensure_tags(self, resource, resource_type=None):
        if not resource_type or not resource:
            self.module.fail_json(msg="Error: Missing resource or resource_type for tags.")

        if 'tags' in resource:
            tags = self.module.params.get('tags')
            if tags is not None:
                self._delete_tags(resource, resource_type, tags)
                self._create_tags(resource, resource_type, tags)
                resource['tags'] = self.get_tags(resource)
        return resource

    def get_capabilities(self, key=None):
        if self.capabilities:
            return self._get_by_key(key, self.capabilities)
        capabilities = self.cs.listCapabilities()
        self.capabilities = capabilities['capability']
        return self._get_by_key(key, self.capabilities)

    # TODO: for backward compatibility only, remove if not used anymore
    def _poll_job(self, job=None, key=None):
        return self.poll_job(job=job, key=key)

    def poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job

    def get_result(self, resource):
        if resource:
            if 'id' in resource:
                self.result['id'] = resource['id']
            if 'project' in resource:
                self.result['project'] = resource['project']
            if 'domain' in resource:
                self.result['domain'] = resource['domain']
            if 'account' in resource:
                self.result['account'] = resource['account']
            if 'zonename' in resource:
                self.result['zone'] = resource['zonename']
        return self.result
<|endoftext|>


<commit_before># Headers
from threading import Thread
import urllib
import json
import redis
from os import path
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import re
import pandas as pd
import prettypandas as pp

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from TwitterAPI import TwitterAPI, TwitterRestPager, TwitterRequestError, TwitterConnectionError
from keys import consumer_key, consumer_secret, access_token_key, access_token_secret

# Instantiate the database
r_server = redis.Redis("localhost")

# Print all the values for a key
def printValues(server, key):
    print server.lrange(key, 0, -1)

# One time results
def singleCall(query='blockchain', count = 50):
    r = api.request('search/tweets', {'q':query, 'count':count })
    for item in r.get_iterator():
        if 'text' in item:
            print item['text']
    return

# Paging the results
def pagingCall(server, query='blockchain', key='Tweets', count = 200, limit = 200):
    # Keep tabs on the number of entries
    numEntries = 1

    r = TwitterRestPager(api, 'search/tweets', {'q':query, 'count':count})
    for item in r.get_iterator():
        if 'text' in item:
            # Pass-in tweets into a list
            server.rpush(key, re.sub(r"http\S+", "", item['text']))

            if(numEntries % 100 == 0):
                print 'Entries so far', numEntries
                #printValues(server, key)
            if(numEntries == limit):
                break

            # Move on to the next entry
            numEntries += 1

        elif 'message' in item and item['code'] == 88:
            print 'SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message']
            break

    return

def iterateTweets(sentiment, server, pager):
    count = 0
    for tweet in pager.get_iterator():
        count += 1
        if 'text' in tweet:
            server.rpush(sentiment, re.sub(r"http\S+", "", item['text']))

            # Print progress periodically
            if(count % 100 == 0):
                print sentiment, count

def joinAndEncode(wordList):
    return urllib.quote_plus(''.join(wordList))

# Paging sentiment comparison
def sentimentComparison(server, query1 = 'python', query2 = 'javascript', count = 50):
    # Keep tabs on the number of entries
    numEntries = 1

    positiveSentiment = ' :)'
    negativeSentiment = ' :('

    h_plus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query1, positiveSentiment]), 'count':count})
    h_minus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query1, negativeSentiment]), 'count':count})
    d_plus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query2, positiveSentiment]), 'count':count})
    d_minus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query2, negativeSentiment]), 'count':count})

    tweets_list = [h_plus, h_minus, d_plus, d_minus]
    tweets_titles = ['h+', 'h-', 'd+', 'd-']
    tweets_lengths = []

    t1 = Thread(target=iterateTweets, args=(tweets_titles[0], r_server, h_plus))
    t2 = Thread(target=iterateTweets, args=(tweets_titles[1], r_server, h_minus))
    t3 = Thread(target=iterateTweets, args=(tweets_titles[2], r_server, d_plus))
    t4 = Thread(target=iterateTweets, args=(tweets_titles[3], r_server, d_minus))

    t1.start()
    t2.start()
    t3.start()
    t4.start()

    print 'sentimentComparison() complete!'

    return

def generateWordCloud(text):
    # read the mask / color image
    # taken from http://jirkavinse.deviantart.com/art/quot-Real-Life-quot-Alice-282261010
    d = path.dirname(__file__)

    cloud_coloring = np.array(Image.open(path.join(d, "us-mask-white.png")))
    stopwords = set(STOPWORDS)
    stopwords.add("said")

    wc = WordCloud(background_color="black", max_words=2000, mask=cloud_coloring,
                   stopwords=stopwords, max_font_size=40, random_state=42)
    # generate word cloud
    wc.generate(text)

    # create coloring from image
    image_colors = ImageColorGenerator(cloud_coloring)

    # show
    plt.imshow(wc)
    plt.axis("off")
    plt.show()

def generateTable(text):
    # Start by getting a frequency dictionary
    d = path.dirname(__file__)

    cloud_coloring = np.array(Image.open(path.join(d, "us-mask-white.png")))
    stopwords = set(STOPWORDS)
    stopwords.add("said")

    wc = WordCloud(background_color="black", max_words=2000, mask=cloud_coloring,
                   stopwords=stopwords, max_font_size=40, random_state=42)

    frequenciesDict = wc.process_text(text)
    frequencies = pd.Series(frequenciesDict)
    frequencies.index.name = 'Word'

    print frequencies.index

    # Make table
    (pp.PrettyPandas(frequencies))

def getTweetsList(server, key):
    # Returns all tweets
    return server.lrange(key, 0, -1)

def getVal(server, key):
    return server.get(key)

def clearList(server, key):
    server.ltrim(key, -1, 0)
    return

def deleteKey(server, key):
    server.delete(key)
    return

api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, 'oAuth2')
<commit_msg>dataframe successfully created<commit_after># Headers
from threading import Thread
import urllib
import json
import redis
from os import path
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import re
import pandas as pd
import prettypandas as pp

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from TwitterAPI import TwitterAPI, TwitterRestPager, TwitterRequestError, TwitterConnectionError
from keys import consumer_key, consumer_secret, access_token_key, access_token_secret

# Instantiate the database
r_server = redis.Redis("localhost")

# Print all the values for a key
def printValues(server, key):
    print server.lrange(key, 0, -1)

# One time results
def singleCall(query='blockchain', count = 50):
    r = api.request('search/tweets', {'q':query, 'count':count })
    for item in r.get_iterator():
        if 'text' in item:
            print item['text']
    return

# Paging the results
def pagingCall(server, query='blockchain', key='Tweets', count = 200, limit = 200):
    # Keep tabs on the number of entries
    numEntries = 1

    r = TwitterRestPager(api, 'search/tweets', {'q':query, 'count':count})
    for item in r.get_iterator():
        if 'text' in item:
            # Pass-in tweets into a list
            server.rpush(key, re.sub(r"http\S+", "", item['text']))

            if(numEntries % 100 == 0):
                print 'Entries so far', numEntries
                #printValues(server, key)
            if(numEntries == limit):
                break

            # Move on to the next entry
            numEntries += 1

        elif 'message' in item and item['code'] == 88:
            print 'SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message']
            break

    return

def iterateTweets(sentiment, server, pager):
    count = 0
    for tweet in pager.get_iterator():
        count += 1
        if 'text' in tweet:
            server.rpush(sentiment, re.sub(r"http\S+", "", item['text']))

            # Print progress periodically
            if(count % 100 == 0):
                print sentiment, count

def joinAndEncode(wordList):
    return urllib.quote_plus(''.join(wordList))

# Paging sentiment comparison
def sentimentComparison(server, query1 = 'python', query2 = 'javascript', count = 50):
    # Keep tabs on the number of entries
    numEntries = 1

    positiveSentiment = ' :)'
    negativeSentiment = ' :('

    h_plus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query1, positiveSentiment]), 'count':count})
    h_minus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query1, negativeSentiment]), 'count':count})
    d_plus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query2, positiveSentiment]), 'count':count})
    d_minus = TwitterRestPager(api, 'search/tweets', {'q':joinAndEncode([query2, negativeSentiment]), 'count':count})

    tweets_list = [h_plus, h_minus, d_plus, d_minus]
    tweets_titles = ['h+', 'h-', 'd+', 'd-']
    tweets_lengths = []

    t1 = Thread(target=iterateTweets, args=(tweets_titles[0], r_server, h_plus))
    t2 = Thread(target=iterateTweets, args=(tweets_titles[1], r_server, h_minus))
    t3 = Thread(target=iterateTweets, args=(tweets_titles[2], r_server, d_plus))
    t4 = Thread(target=iterateTweets, args=(tweets_titles[3], r_server, d_minus))

    t1.start()
    t2.start()
    t3.start()
    t4.start()

    print 'sentimentComparison() complete!'

    return

def generateWordCloud(text):
    # read the mask / color image
    # taken from http://jirkavinse.deviantart.com/art/quot-Real-Life-quot-Alice-282261010
    d = path.dirname(__file__)

    cloud_coloring = np.array(Image.open(path.join(d, "us-mask-white.png")))
    stopwords = set(STOPWORDS)
    stopwords.add("said")

    wc = WordCloud(background_color="black", max_words=2000, mask=cloud_coloring,
                   stopwords=stopwords, max_font_size=40, random_state=42)
    # generate word cloud
    wc.generate(text)

    # create coloring from image
    image_colors = ImageColorGenerator(cloud_coloring)

    # show
    plt.imshow(wc)
    plt.axis("off")
    plt.show()

def generateTable(text):
    # Start by getting a frequency dictionary
    d = path.dirname(__file__)

    cloud_coloring = np.array(Image.open(path.join(d, "us-mask-white.png")))
    stopwords = set(STOPWORDS)
    stopwords.add("said")

    wc = WordCloud(background_color="black", max_words=2000, mask=cloud_coloring,
                   stopwords=stopwords, max_font_size=40, random_state=42)

    frequenciesDict = wc.process_text(text)

    words = frequenciesDict.keys()
    freq = frequenciesDict.values()

    frequencies = pd.DataFrame({ 'words' : words, 'frequencies' : freq })

    print frequencies

    # Make table
    # (pp.PrettyPandas(frequencies).total())

def getTweetsList(server, key):
    # Returns all tweets
    return server.lrange(key, 0, -1)

def getVal(server, key):
    return server.get(key)

def clearList(server, key):
    server.ltrim(key, -1, 0)
    return

def deleteKey(server, key):
    server.delete(key)
    return

api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, 'oAuth2')
<|endoftext|>