<commit_before># -*- coding: utf-8 -*-
"""
    EauDouce.views.front
    ~~~~~~~~~~~~~~

    Foreground view.

    :copyright: (c) 2017 by Mr.tao.
    :license: MIT, see LICENSE for more details.
"""

import datetime
from urlparse import urljoin
from werkzeug.contrib.atom import AtomFeed
from flask import Blueprint, g, render_template, request, redirect, url_for, make_response, abort
from utils.tool import login_required, logger, BaiduActivePush

front_blueprint = Blueprint("front", __name__)

@front_blueprint.route("/")
def index():
    return render_template("front/blogIndex.html")

@front_blueprint.route('/blog/<int:bid>.html')
def blogShow(bid):
    data = g.api.blog_get_id(bid).get("data")
    if data:
        BaiduActivePushResult = False
        if g.plugins['BaiduActivePush']['enable'] in ("true", "True", True):
            BaiduActivePushResult = True if BaiduActivePush(request.url, original=True if data.get("sources") == "原创" else False).get("success") == 1 else False
        return render_template("front/blogShow.html", blogId=bid, data=data, BaiduActivePushResult=BaiduActivePushResult)
    else:
        return abort(404)

@front_blueprint.route("/blog/write/")
@login_required
def blogWrite():
    return render_template("front/blogWrite.html")

@front_blueprint.route("/blog/edit/")
@login_required
def blogEdit():
    blogId = request.args.get("blogId")
    if blogId:
        data = g.api.blog_get_id(blogId).get("data")
        if data:
            if g.username == data.get("author") or g.username in g.api.user_get_admins().get("data"):
                return render_template("front/blogEdit.html", blogId=blogId, data=data)
        return abort(404)

@front_blueprint.route("/blog/resource/")
def blogResource():
    return render_template("front/blogResource.html")

@front_blueprint.route("/blog/search/")
def blogSearch():
    return render_template("front/blogSearch.html")

@front_blueprint.route("/user/<user>/")
def userHome(user=None):
    return render_template("front/userHome.html", user=user)

@front_blueprint.route("/user/ChangeAvater/")
@login_required
def userChangeAvater():
    return render_template("front/userChangeAvater.html")

@front_blueprint.route("/user/ChangePassword/")
@login_required
def userChangePassword():
    return render_template("front/userChangePassword.html")

@front_blueprint.route("/user/ChangeCover/")
@login_required
def userChangeCover():
    return render_template("front/userChangeCover2.html")
    #return render_template("front/userChangeCover.html")

@front_blueprint.route("/user/ChangeProfile/")
@login_required
def userChangeProfile():
    return render_template("front/userChangeProfile.html")

@front_blueprint.route("/robots.txt")
def robots():
    return """
User-agent: *
Disallow:
    """

@front_blueprint.route("/sitemap.xml")
def sitemapxml():
    resp = make_response(render_template("public/sitemap.xml"))
    resp.headers["Content-Type"] = "application/xml"
    return resp

@front_blueprint.route("/sitemap.html")
def sitemaphtml():
    return render_template("public/sitemap.html")

@front_blueprint.route("/feed/")
def feed():
    data = g.api.blog_get_all(limit=10).get("data")
    feed = AtomFeed(g.api.get_sys_config().get("data").get("site_feedname"), feed_url=request.url, url=request.url_root, subtitle="From the latest article in {}".format(request.url))
    for article in data:
        updated = article['update_time'][:10] if article['update_time'] else article['create_time'][:10]
        feed.add(article['title'], unicode(article['content']),
                 content_type='html',
                 author=article['author'],
                 id=article['id'],
                 url=urljoin(request.url_root, url_for(".blogShow", bid=article['id'])),
                 updated=datetime.datetime.strptime(updated, "%Y-%m-%d"),
                 published=datetime.datetime.strptime(article['create_time'][:10], "%Y-%m-%d"))
    return feed.get_response()
<commit_msg>上传头像裁剪<commit_after># -*- coding: utf-8 -*-
"""
    EauDouce.views.front
    ~~~~~~~~~~~~~~

    Foreground view.

    :copyright: (c) 2017 by Mr.tao.
    :license: MIT, see LICENSE for more details.
"""

import datetime
from urlparse import urljoin
from werkzeug.contrib.atom import AtomFeed
from flask import Blueprint, g, render_template, request, redirect, url_for, make_response, abort
from utils.tool import login_required, logger, BaiduActivePush

front_blueprint = Blueprint("front", __name__)

@front_blueprint.route("/")
def index():
    return render_template("front/blogIndex.html")

@front_blueprint.route('/blog/<int:bid>.html')
def blogShow(bid):
    data = g.api.blog_get_id(bid).get("data")
    if data:
        BaiduActivePushResult = False
        if g.plugins['BaiduActivePush']['enable'] in ("true", "True", True):
            BaiduActivePushResult = True if BaiduActivePush(request.url, original=True if data.get("sources") == "原创" else False).get("success") == 1 else False
        return render_template("front/blogShow.html", blogId=bid, data=data, BaiduActivePushResult=BaiduActivePushResult)
    else:
        return abort(404)

@front_blueprint.route("/blog/write/")
@login_required
def blogWrite():
    return render_template("front/blogWrite.html")

@front_blueprint.route("/blog/edit/")
@login_required
def blogEdit():
    blogId = request.args.get("blogId")
    if blogId:
        data = g.api.blog_get_id(blogId).get("data")
        if data:
            if g.username == data.get("author") or g.username in g.api.user_get_admins().get("data"):
                return render_template("front/blogEdit.html", blogId=blogId, data=data)
        return abort(404)

@front_blueprint.route("/blog/resource/")
def blogResource():
    return render_template("front/blogResource.html")

@front_blueprint.route("/blog/search/")
def blogSearch():
    return render_template("front/blogSearch.html")

@front_blueprint.route("/user/<user>/")
def userHome(user=None):
    return render_template("front/userHome.html", user=user)

@front_blueprint.route("/user/ChangeAvater/")
@login_required
def userChangeAvater():
    return render_template("front/userChangeAvater2.html")
    return render_template("front/userChangeAvater.html")

@front_blueprint.route("/user/ChangePassword/")
@login_required
def userChangePassword():
    return render_template("front/userChangePassword.html")

@front_blueprint.route("/user/ChangeCover/")
@login_required
def userChangeCover():
    return render_template("front/userChangeCover.html")

@front_blueprint.route("/user/ChangeProfile/")
@login_required
def userChangeProfile():
    return render_template("front/userChangeProfile.html")

@front_blueprint.route("/robots.txt")
def robots():
    return """
User-agent: *
Disallow:
    """

@front_blueprint.route("/sitemap.xml")
def sitemapxml():
    resp = make_response(render_template("public/sitemap.xml"))
    resp.headers["Content-Type"] = "application/xml"
    return resp

@front_blueprint.route("/sitemap.html")
def sitemaphtml():
    return render_template("public/sitemap.html")

@front_blueprint.route("/feed/")
def feed():
    data = g.api.blog_get_all(limit=10).get("data")
    feed = AtomFeed(g.api.get_sys_config().get("data").get("site_feedname"), feed_url=request.url, url=request.url_root, subtitle="From the latest article in {}".format(request.url))
    for article in data:
        updated = article['update_time'][:10] if article['update_time'] else article['create_time'][:10]
        feed.add(article['title'], unicode(article['content']),
                 content_type='html',
                 author=article['author'],
                 id=article['id'],
                 url=urljoin(request.url_root, url_for(".blogShow", bid=article['id'])),
                 updated=datetime.datetime.strptime(updated, "%Y-%m-%d"),
                 published=datetime.datetime.strptime(article['create_time'][:10], "%Y-%m-%d"))
    return feed.get_response()
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import re
import json
import time

from django.core.management.base import BaseCommand

from programs.models import Program, ProgramModules, LearningPlan
from disciplines.models import Discipline, Semester, TrainingTerms
from modules.models import Module

class Command(BaseCommand):
    help = "Create Django objects from ITOO-UrFU/EPP-generated json."
    requires_system_checks = True
    requires_migrations_checks = True

    def add_arguments(self, parser):
        parser.add_argument('epp_json', nargs=1)
        parser.add_argument('program_title', nargs=1)

    def handle(self, *args, **options):
        epp_json = options["epp_json"][0]
        program_title = options["program_title"][0]

        with open(epp_json, encoding='utf-8') as epp_json:
            epp = json.loads('\n'.join(epp_json.readlines()))

        try:
            program = Program.objects.filter(title=program_title).first()
            program.status = "p"
            program.save()
        except:
            raise NotImplementedError

        stage = False if "stage" not in epp.keys() else True if "утверждено" in epp["stage"].lower() else False
        displayableTitle = None if "version" not in epp.keys() else epp["version"]
        number = None if "number" not in epp.keys() else epp["number"]
        active = None if "active" not in epp.keys() else epp["active"]
        title = None if "title" not in epp.keys() else epp["title"]

        term = None if "term" not in epp.keys() else epp["term"]

        lps = LearningPlan.objects.filter(uni_number=number, status="p")
        if len(lps) > 0:
            for lp in lps:
                lp.uni_displayableTitle = displayableTitle
                lp.uni_number = number
                lp.uni_active = active
                lp.uni_title = title
                lp.uni_stage = stage
                lp.uni_html = json.dumps(epp)
                lp.save()
                if lp not in program.learning_plans.all():
                    program.learning_plans.add(lp)
                    program.save()
        else:
            lp = LearningPlan(uni_displayableTitle=displayableTitle,
                              uni_number=number,
                              uni_active=active,
                              uni_title=title,
                              uni_stage=stage,
                              uni_html=json.dumps(epp),
                              status="p"
                              )
            lp.save()
            print(lp.id)
            program.learning_plans.add(lp)
            program.save()

            # Добавили учебный план

        print("Semesters count: ", term)

        for epp_module in epp["modules"]:
            module_obj, semester = self.create_module(epp_module, program, term)

    @staticmethod
    def create_module(epp_module, program, term):
        semester = min([int(d["firstSemester"]) for d in epp_module["disciplines"]])
        module_obj = Module.objects.filter(uni_number=epp_module["disciplineNumberheaderCell"]).first()

        if module_obj and term == 8:
            module_obj.uni_uuid = epp_module["uuid"]
            module_obj.uni_number = epp_module["disciplineNumberheaderCell"]
            module_obj.uni_coordinator = epp_module["coordinator"]
            module_obj.uni_type = epp_module["type"]
            module_obj.uni_title = epp_module["title"]
            module_obj.uni_testUnits = epp_module["testUnits"]
            module_obj.uni_priority = epp_module["priority"]
            module_obj.uni_state = epp_module["state"]
            module_obj.uni_comment = epp_module["comment"]
            module_obj.uni_file = epp_module["file"]
            module_obj.uni_specialities = epp_module["specialities"]
            module_obj.program = program
            module_obj.semester = semester
            module_obj.status = 'p'
            module_obj.save()

            program_module = ProgramModules.objects.filter(program=program, module=module_obj)
            if not program_module:
                program_module = ProgramModules(program=program, module=module_obj, semester=module_obj.semester,
                                                status="p", index=epp_module["indexheaderCell"])
                program_module.save()

            for_delete = []

            for epp_discipline in epp_module["disciplines"]:
                print("first", epp_discipline['titleheaderCell'])
                discipline = Discipline.objects.filter(module=module_obj,
                                                       title=epp_discipline['titleheaderCell']).first()
                if not discipline and any(ext in epp_discipline["titleheaderCell"] for ext in ["I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X", "XI", "XII", "XIII", "XIV", "XV"]):
                    discipline = Discipline.objects.filter(module=module_obj,
                                                           title__contains=epp_discipline['titleheaderCell']).first()
                    print(epp_discipline['titleheaderCell'])
                    for_delete.append(discipline.id)

                training_semester = int(epp_discipline["firstSemester"])

                try:
                    if epp_discipline["exam"] > epp_discipline["credit"]:
                        form = "e"
                    else:
                        form = "z"
                except:
                    print(epp_discipline)

                parted_discipline = Discipline.objects.filter(title=epp_discipline['titleheaderCell'],
                                                              module=module_obj, status="p",
                                                              archived=False).first()

                if not parted_discipline:
                    parted_discipline = Discipline.objects.create(
                        title=epp_discipline['titleheaderCell'],
                        description=discipline.description,
                        module=module_obj,
                        labor=epp_discipline["gosLoadInTestUnitsheaderCell"],
                        period=training_semester - semester + 1,
                        form=form,
                        uni_uid=discipline.uni_uid,
                        uni_discipline=discipline.uni_discipline,
                        uni_number=discipline.uni_number,
                        uni_section=discipline.uni_section,
                        uni_file=discipline.uni_file,
                        status="p"
                    )

                if term == 8:
                    cur_term = TrainingTerms.objects.filter(title="4 года").first()
                elif term == 10:
                    cur_term = TrainingTerms.objects.filter(title="5 лет").first()
                elif term == 7:
                    cur_term = TrainingTerms.objects.filter(title="3,5 года").first()

                semester_obj = Semester.objects.filter(discipline=discipline, training_semester=training_semester,
                                                       program=program, term=cur_term).first()
                if not semester_obj:
                    semester_obj = Semester.objects.create(discipline=discipline,
                                                           training_semester=training_semester,
                                                           program=program,
                                                           year='2017',
                                                           admission_semester="0",
                                                           term=cur_term,
                                                           )

            Discipline.objects.filter(id__in=for_delete).delete()
        else:
            print("Модуль не найден! Загрузите новую версию modules.json")

        return module_obj, semester
<commit_msg>epp load#56<commit_after>from bs4 import BeautifulSoup
import re
import json
import time

from django.core.management.base import BaseCommand

from programs.models import Program, ProgramModules, LearningPlan
from disciplines.models import Discipline, Semester, TrainingTerms
from modules.models import Module

class Command(BaseCommand):
    help = "Create Django objects from ITOO-UrFU/EPP-generated json."
    requires_system_checks = True
    requires_migrations_checks = True

    def add_arguments(self, parser):
        parser.add_argument('epp_json', nargs=1)
        parser.add_argument('program_title', nargs=1)

    def handle(self, *args, **options):
        epp_json = options["epp_json"][0]
        program_title = options["program_title"][0]

        with open(epp_json, encoding='utf-8') as epp_json:
            epp = json.loads('\n'.join(epp_json.readlines()))

        try:
            program = Program.objects.filter(title=program_title).first()
            program.status = "p"
            program.save()
        except:
            raise NotImplementedError

        stage = False if "stage" not in epp.keys() else True if "утверждено" in epp["stage"].lower() else False
        displayableTitle = None if "version" not in epp.keys() else epp["version"]
        number = None if "number" not in epp.keys() else epp["number"]
        active = None if "active" not in epp.keys() else epp["active"]
        title = None if "title" not in epp.keys() else epp["title"]

        term = None if "term" not in epp.keys() else epp["term"]

        lps = LearningPlan.objects.filter(uni_number=number, status="p")
        if len(lps) > 0:
            for lp in lps:
                lp.uni_displayableTitle = displayableTitle
                lp.uni_number = number
                lp.uni_active = active
                lp.uni_title = title
                lp.uni_stage = stage
                lp.uni_html = json.dumps(epp)
                lp.save()
                if lp not in program.learning_plans.all():
                    program.learning_plans.add(lp)
                    program.save()
        else:
            lp = LearningPlan(uni_displayableTitle=displayableTitle,
                              uni_number=number,
                              uni_active=active,
                              uni_title=title,
                              uni_stage=stage,
                              uni_html=json.dumps(epp),
                              status="p"
                              )
            lp.save()
            print(lp.id)
            program.learning_plans.add(lp)
            program.save()

            # Добавили учебный план

        print("Semesters count: ", term)

        for epp_module in epp["modules"]:
            module_obj, semester = self.create_module(epp_module, program, term)

    @staticmethod
    def create_module(epp_module, program, term):
        semester = min([int(d["firstSemester"]) for d in epp_module["disciplines"]])
        module_obj = Module.objects.filter(uni_number=epp_module["disciplineNumberheaderCell"]).first()

        if module_obj and term == 8:
            module_obj.uni_uuid = epp_module["uuid"]
            module_obj.uni_number = epp_module["disciplineNumberheaderCell"]
            module_obj.uni_coordinator = epp_module["coordinator"]
            module_obj.uni_type = epp_module["type"]
            module_obj.uni_title = epp_module["title"]
            module_obj.uni_testUnits = epp_module["testUnits"]
            module_obj.uni_priority = epp_module["priority"]
            module_obj.uni_state = epp_module["state"]
            module_obj.uni_comment = epp_module["comment"]
            module_obj.uni_file = epp_module["file"]
            module_obj.uni_specialities = epp_module["specialities"]
            module_obj.program = program
            module_obj.semester = semester
            module_obj.status = 'p'
            module_obj.save()

            program_module = ProgramModules.objects.filter(program=program, module=module_obj)
            if not program_module:
                program_module = ProgramModules(program=program, module=module_obj, semester=module_obj.semester,
                                                status="p", index=epp_module["indexheaderCell"])
                program_module.save()

            for_delete = []

            for epp_discipline in epp_module["disciplines"]:
                print("first", epp_discipline['titleheaderCell'])
                discipline = Discipline.objects.filter(module=module_obj,
                                                       title=epp_discipline['titleheaderCell']).first()
                print(any(ext in epp_discipline["titleheaderCell"] for ext in ["I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X", "XI", "XII", "XIII", "XIV", "XV"]))
                if not discipline and any(ext in epp_discipline["titleheaderCell"] for ext in ["I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X", "XI", "XII", "XIII", "XIV", "XV"]):
                    discipline = Discipline.objects.filter(module=module_obj,
                                                           title__contains=epp_discipline['titleheaderCell']).first()
                    print(epp_discipline['titleheaderCell'])
                    for_delete.append(discipline.id)

                training_semester = int(epp_discipline["firstSemester"])

                try:
                    if epp_discipline["exam"] > epp_discipline["credit"]:
                        form = "e"
                    else:
                        form = "z"
                except:
                    print(epp_discipline)

                parted_discipline = Discipline.objects.filter(title=epp_discipline['titleheaderCell'],
                                                              module=module_obj, status="p",
                                                              archived=False).first()

                if not parted_discipline:
                    parted_discipline = Discipline.objects.create(
                        title=epp_discipline['titleheaderCell'],
                        description=discipline.description,
                        module=module_obj,
                        labor=epp_discipline["gosLoadInTestUnitsheaderCell"],
                        period=training_semester - semester + 1,
                        form=form,
                        uni_uid=discipline.uni_uid,
                        uni_discipline=discipline.uni_discipline,
                        uni_number=discipline.uni_number,
                        uni_section=discipline.uni_section,
                        uni_file=discipline.uni_file,
                        status="p"
                    )

                if term == 8:
                    cur_term = TrainingTerms.objects.filter(title="4 года").first()
                elif term == 10:
                    cur_term = TrainingTerms.objects.filter(title="5 лет").first()
                elif term == 7:
                    cur_term = TrainingTerms.objects.filter(title="3,5 года").first()

                semester_obj = Semester.objects.filter(discipline=discipline, training_semester=training_semester,
                                                       program=program, term=cur_term).first()
                if not semester_obj:
                    semester_obj = Semester.objects.create(discipline=discipline,
                                                           training_semester=training_semester,
                                                           program=program,
                                                           year='2017',
                                                           admission_semester="0",
                                                           term=cur_term,
                                                           )

            Discipline.objects.filter(id__in=for_delete).delete()
        else:
            print("Модуль не найден! Загрузите новую версию modules.json")

        return module_obj, semester
<|endoftext|>


<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import atexit
import logging

from pylib.device import device_errors
from pylib.device import device_utils

class PerfControl(object):
  """Provides methods for setting the performance mode of a device."""
  _CPU_PATH = '/sys/devices/system/cpu'
  _KERNEL_MAX = '/sys/devices/system/cpu/kernel_max'

  def __init__(self, device):
    self._device = device
    # this will raise an AdbCommandFailedError if no CPU files are found
    self._cpu_files = self._device.RunShellCommand(
        'ls -d cpu[0-9]*', cwd=self._CPU_PATH, check_return=True, as_root=True)
    assert self._cpu_files, 'Failed to detect CPUs.'
    self._cpu_file_list = ' '.join(self._cpu_files)
    logging.info('CPUs found: %s', self._cpu_file_list)
    self._have_mpdecision = self._device.FileExists('/system/bin/mpdecision')

  def SetHighPerfMode(self):
    """Sets the highest stable performance mode for the device."""
    try:
      self._device.EnableRoot()
    except device_errors.CommandFailedError:
      message = 'Need root for performance mode. Results may be NOISY!!'
      logging.warning(message)
      # Add an additional warning at exit, such that it's clear that any results
      # may be different/noisy (due to the lack of intended performance mode).
      atexit.register(logging.warning, message)
      return

    product_model = self._device.product_model
    # TODO(epenner): Enable on all devices (http://crbug.com/383566)
    if 'Nexus 4' == product_model:
      self._ForceAllCpusOnline(True)
      if not self._AllCpusAreOnline():
        logging.warning('Failed to force CPUs online. Results may be NOISY!')
      self._SetScalingGovernorInternal('performance')
    elif 'Nexus 5' == product_model:
      self._ForceAllCpusOnline(True)
      if not self._AllCpusAreOnline():
        logging.warning('Failed to force CPUs online. Results may be NOISY!')
      self._SetScalingGovernorInternal('performance')
      self._SetScalingMaxFreq(1190400)
      self._SetMaxGpuClock(200000000)
    else:
      self._SetScalingGovernorInternal('performance')

  def SetPerfProfilingMode(self):
    """Enables all cores for reliable perf profiling."""
    self._ForceAllCpusOnline(True)
    self._SetScalingGovernorInternal('performance')
    if not self._AllCpusAreOnline():
      if not self._device.HasRoot():
        raise RuntimeError('Need root to force CPUs online.')
      raise RuntimeError('Failed to force CPUs online.')

  def SetDefaultPerfMode(self):
    """Sets the performance mode for the device to its default mode."""
    if not self._device.HasRoot():
      return
    product_model = self._device.product_model
    if 'Nexus 5' == product_model:
      if self._AllCpusAreOnline():
        self._SetScalingMaxFreq(2265600)
        self._SetMaxGpuClock(450000000)

    governor_mode = {
        'GT-I9300': 'pegasusq',
        'Galaxy Nexus': 'interactive',
        'Nexus 4': 'ondemand',
        'Nexus 5': 'ondemand',
        'Nexus 7': 'interactive',
        'Nexus 10': 'interactive'
    }.get(product_model, 'ondemand')
    self._SetScalingGovernorInternal(governor_mode)
    self._ForceAllCpusOnline(False)

  def GetCpuInfo(self):
    online = (output.rstrip() == '1' and status == 0
              for (_, output, status) in self._ForEachCpu('cat "$CPU/online"'))
    governor = (output.rstrip() if status == 0 else None
                for (_, output, status)
                in self._ForEachCpu('cat "$CPU/cpufreq/scaling_governor"'))
    return zip(self._cpu_files, online, governor)

  def _ForEachCpu(self, cmd):
    script = '; '.join([
        'for CPU in %s' % self._cpu_file_list,
        'do %s' % cmd,
        'echo -n "%~%$?%~%"',
        'done'
    ])
    output = self._device.RunShellCommand(
        script, cwd=self._CPU_PATH, check_return=True, as_root=True)
    output = '\n'.join(output).split('%~%')
    return zip(self._cpu_files, output[0::2], (int(c) for c in output[1::2]))

  def _WriteEachCpuFile(self, path, value):
    results = self._ForEachCpu(
        'test -e "$CPU/{path}" && echo {value} > "$CPU/{path}"'.format(
            path=path, value=value))
    cpus = ' '.join(cpu for (cpu, _, status) in results if status == 0)
    if cpus:
      logging.info('Successfully set %s to %r on: %s', path, value, cpus)
    else:
      logging.warning('Failed to set %s to %r on any cpus')

  def _SetScalingGovernorInternal(self, value):
    self._WriteEachCpuFile('cpufreq/scaling_governor', value)

  def _SetScalingMaxFreq(self, value):
    self._WriteEachCpuFile('cpufreq/scaling_max_freq', '%d' % value)

  def _SetMaxGpuClock(self, value):
    self._device.WriteFile('/sys/class/kgsl/kgsl-3d0/max_gpuclk',
                           str(value),
                           as_root=True)

  def _AllCpusAreOnline(self):
    results = self._ForEachCpu('cat "$CPU/online"')
    # TODO(epenner): Investigate why file may be missing
    # (http://crbug.com/397118)
    return all(output.rstrip() == '1' and status == 0
               for (cpu, output, status) in results
               if cpu != 'cpu0')

  def _ForceAllCpusOnline(self, force_online):
    """Enable all CPUs on a device.

    Some vendors (or only Qualcomm?) hot-plug their CPUs, which can add noise
    to measurements:
    - In perf, samples are only taken for the CPUs that are online when the
      measurement is started.
    - The scaling governor can't be set for an offline CPU and frequency scaling
      on newly enabled CPUs adds noise to both perf and tracing measurements.

    It appears Qualcomm is the only vendor that hot-plugs CPUs, and on Qualcomm
    this is done by "mpdecision".

    """
    if self._have_mpdecision:
      script = 'stop mpdecision' if force_online else 'start mpdecision'
      self._device.RunShellCommand(script, check_return=True, as_root=True)

    if not self._have_mpdecision and not self._AllCpusAreOnline():
      logging.warning('Unexpected cpu hot plugging detected.')

    if force_online:
      self._ForEachCpu('echo 1 > "$CPU/online"')
<commit_msg>[Android] Fix warning message in perf_control.py<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import atexit
import logging

from pylib.device import device_errors
from pylib.device import device_utils

class PerfControl(object):
  """Provides methods for setting the performance mode of a device."""
  _CPU_PATH = '/sys/devices/system/cpu'
  _KERNEL_MAX = '/sys/devices/system/cpu/kernel_max'

  def __init__(self, device):
    self._device = device
    # this will raise an AdbCommandFailedError if no CPU files are found
    self._cpu_files = self._device.RunShellCommand(
        'ls -d cpu[0-9]*', cwd=self._CPU_PATH, check_return=True, as_root=True)
    assert self._cpu_files, 'Failed to detect CPUs.'
    self._cpu_file_list = ' '.join(self._cpu_files)
    logging.info('CPUs found: %s', self._cpu_file_list)
    self._have_mpdecision = self._device.FileExists('/system/bin/mpdecision')

  def SetHighPerfMode(self):
    """Sets the highest stable performance mode for the device."""
    try:
      self._device.EnableRoot()
    except device_errors.CommandFailedError:
      message = 'Need root for performance mode. Results may be NOISY!!'
      logging.warning(message)
      # Add an additional warning at exit, such that it's clear that any results
      # may be different/noisy (due to the lack of intended performance mode).
      atexit.register(logging.warning, message)
      return

    product_model = self._device.product_model
    # TODO(epenner): Enable on all devices (http://crbug.com/383566)
    if 'Nexus 4' == product_model:
      self._ForceAllCpusOnline(True)
      if not self._AllCpusAreOnline():
        logging.warning('Failed to force CPUs online. Results may be NOISY!')
      self._SetScalingGovernorInternal('performance')
    elif 'Nexus 5' == product_model:
      self._ForceAllCpusOnline(True)
      if not self._AllCpusAreOnline():
        logging.warning('Failed to force CPUs online. Results may be NOISY!')
      self._SetScalingGovernorInternal('performance')
      self._SetScalingMaxFreq(1190400)
      self._SetMaxGpuClock(200000000)
    else:
      self._SetScalingGovernorInternal('performance')

  def SetPerfProfilingMode(self):
    """Enables all cores for reliable perf profiling."""
    self._ForceAllCpusOnline(True)
    self._SetScalingGovernorInternal('performance')
    if not self._AllCpusAreOnline():
      if not self._device.HasRoot():
        raise RuntimeError('Need root to force CPUs online.')
      raise RuntimeError('Failed to force CPUs online.')

  def SetDefaultPerfMode(self):
    """Sets the performance mode for the device to its default mode."""
    if not self._device.HasRoot():
      return
    product_model = self._device.product_model
    if 'Nexus 5' == product_model:
      if self._AllCpusAreOnline():
        self._SetScalingMaxFreq(2265600)
        self._SetMaxGpuClock(450000000)

    governor_mode = {
        'GT-I9300': 'pegasusq',
        'Galaxy Nexus': 'interactive',
        'Nexus 4': 'ondemand',
        'Nexus 5': 'ondemand',
        'Nexus 7': 'interactive',
        'Nexus 10': 'interactive'
    }.get(product_model, 'ondemand')
    self._SetScalingGovernorInternal(governor_mode)
    self._ForceAllCpusOnline(False)

  def GetCpuInfo(self):
    online = (output.rstrip() == '1' and status == 0
              for (_, output, status) in self._ForEachCpu('cat "$CPU/online"'))
    governor = (output.rstrip() if status == 0 else None
                for (_, output, status)
                in self._ForEachCpu('cat "$CPU/cpufreq/scaling_governor"'))
    return zip(self._cpu_files, online, governor)

  def _ForEachCpu(self, cmd):
    script = '; '.join([
        'for CPU in %s' % self._cpu_file_list,
        'do %s' % cmd,
        'echo -n "%~%$?%~%"',
        'done'
    ])
    output = self._device.RunShellCommand(
        script, cwd=self._CPU_PATH, check_return=True, as_root=True)
    output = '\n'.join(output).split('%~%')
    return zip(self._cpu_files, output[0::2], (int(c) for c in output[1::2]))

  def _WriteEachCpuFile(self, path, value):
    results = self._ForEachCpu(
        'test -e "$CPU/{path}" && echo {value} > "$CPU/{path}"'.format(
            path=path, value=value))
    cpus = ' '.join(cpu for (cpu, _, status) in results if status == 0)
    if cpus:
      logging.info('Successfully set %s to %r on: %s', path, value, cpus)
    else:
      logging.warning('Failed to set %s to %r on any cpus', path, value)

  def _SetScalingGovernorInternal(self, value):
    self._WriteEachCpuFile('cpufreq/scaling_governor', value)

  def _SetScalingMaxFreq(self, value):
    self._WriteEachCpuFile('cpufreq/scaling_max_freq', '%d' % value)

  def _SetMaxGpuClock(self, value):
    self._device.WriteFile('/sys/class/kgsl/kgsl-3d0/max_gpuclk',
                           str(value),
                           as_root=True)

  def _AllCpusAreOnline(self):
    results = self._ForEachCpu('cat "$CPU/online"')
    # TODO(epenner): Investigate why file may be missing
    # (http://crbug.com/397118)
    return all(output.rstrip() == '1' and status == 0
               for (cpu, output, status) in results
               if cpu != 'cpu0')

  def _ForceAllCpusOnline(self, force_online):
    """Enable all CPUs on a device.

    Some vendors (or only Qualcomm?) hot-plug their CPUs, which can add noise
    to measurements:
    - In perf, samples are only taken for the CPUs that are online when the
      measurement is started.
    - The scaling governor can't be set for an offline CPU and frequency scaling
      on newly enabled CPUs adds noise to both perf and tracing measurements.

    It appears Qualcomm is the only vendor that hot-plugs CPUs, and on Qualcomm
    this is done by "mpdecision".

    """
    if self._have_mpdecision:
      script = 'stop mpdecision' if force_online else 'start mpdecision'
      self._device.RunShellCommand(script, check_return=True, as_root=True)

    if not self._have_mpdecision and not self._AllCpusAreOnline():
      logging.warning('Unexpected cpu hot plugging detected.')

    if force_online:
      self._ForEachCpu('echo 1 > "$CPU/online"')
<|endoftext|>


<commit_before># This file is part of QuTiP: Quantum Toolbox in Python.
#
#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
#    All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without
#    modification, are permitted provided that the following conditions are
#    met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
#       of its contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
###############################################################################

"""
Command line output of information on QuTiP and
dependencies.
"""

import sys
import os
import platform
import numpy
import scipy
import qutip.settings
from qutip import __version__ as qutip_version

def about():
    """
    About box for qutip. Gives version numbers for
    QuTiP, NumPy, SciPy, Cython, and MatPlotLib.
    """
    print("")
    print("QuTiP: Quantum Toolbox in Python")
    print("Copyright (c) 2011 and later.")
    print("Paul D. Nation & Robert J. Johansson")
    print("")
    print("QuTiP Version:      %s" % qutip.__version__)
    print("Numpy Version:      %s" % numpy.__version__)
    print("Scipy Version:      %s" % scipy.__version__)
    try:
        import Cython
        cython_ver = Cython.__version__
    except:
        cython_ver = 'None'
    print("Cython Version:     %s" % cython_ver)
    try:
        import matplotlib
        matplotlib_ver = matplotlib.__version__
    except:
        matplotlib_ver = 'None'
    print("Matplotlib Version: %s" % matplotlib_ver)
    print("Fortran mcsolver:   %s" % str(qutip.settings.fortran))
    print("scikits.umfpack:    %s" % str(qutip.settings.umfpack))
    print("Python Version:     %d.%d.%d" % sys.version_info[0:3])
    print("Platform Info:      %s (%s)" % (platform.system(),
                                           platform.machine()))
    print("")

if __name__ == "__main__":
    about()
<commit_msg>add installation path to info reported by about<commit_after># This file is part of QuTiP: Quantum Toolbox in Python.
#
#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
#    All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without
#    modification, are permitted provided that the following conditions are
#    met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
#       of its contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
###############################################################################

"""
Command line output of information on QuTiP and
dependencies.
"""

import sys
import os
import platform
import numpy
import scipy
import inspect
import qutip.settings
from qutip import __version__ as qutip_version

def about():
    """
    About box for qutip. Gives version numbers for
    QuTiP, NumPy, SciPy, Cython, and MatPlotLib.
    """
    print("")
    print("QuTiP: Quantum Toolbox in Python")
    print("Copyright (c) 2011 and later.")
    print("Paul D. Nation & Robert J. Johansson")
    print("")
    print("QuTiP Version:      %s" % qutip.__version__)
    print("Numpy Version:      %s" % numpy.__version__)
    print("Scipy Version:      %s" % scipy.__version__)
    try:
        import Cython
        cython_ver = Cython.__version__
    except:
        cython_ver = 'None'
    print("Cython Version:     %s" % cython_ver)
    try:
        import matplotlib
        matplotlib_ver = matplotlib.__version__
    except:
        matplotlib_ver = 'None'
    print("Matplotlib Version: %s" % matplotlib_ver)
    print("Fortran mcsolver:   %s" % str(qutip.settings.fortran))
    print("scikits.umfpack:    %s" % str(qutip.settings.umfpack))
    print("Python Version:     %d.%d.%d" % sys.version_info[0:3])
    print("Platform Info:      %s (%s)" % (platform.system(),
                                           platform.machine()))
    qutip_install_path = os.path.dirname(inspect.getsourcefile(qutip))
    print("Installation path:  %s" % qutip_install_path)
    print("")

if __name__ == "__main__":
    about()
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

from os import path

from PyInquirer import style_from_dict, Token, prompt
from pyfiglet import Figlet
from termcolor import colored
from prompt_toolkit.validation import ValidationError, Validator
from re import compile, IGNORECASE
from os.path import exists, splitext
from yaml import safe_load, dump

if exists('thingsboard_gateway/config/tb_gateway.yaml'):
    CONFIG_PATH = 'thingsboard_gateway/config/tb_gateway.yaml'
elif exists(path.dirname(path.abspath(__file__)) + '/config/tb_gateway.yaml'.replace('/', path.sep)):
    CONFIG_PATH = path.dirname(path.abspath(__file__)) + '/config/tb_gateway.yaml'.replace('/', path.sep)
elif exists("/etc/thingsboard-gateway/config/tb_gateway.yaml".replace('/', path.sep)):
    CONFIG_PATH = "/etc/thingsboard-gateway/config/tb_gateway.yaml".replace('/', path.sep)

style = style_from_dict({
    Token.Separator: '#cc5454',
    Token.QuestionMark: '#673ab7 bold',
    Token.Selected: '#673ab7',
    Token.Pointer: '#673ab7 bold',
    Token.Instruction: '',  # default
    Token.Answer: '#673ab7 bold',
    Token.Question: '',
})

class NotNullValidator(Validator):
    def validate(self, document):
        if not document.text or document.text == '':
            raise ValidationError(message='Value can be empty!', cursor_position=len(document.text))

class NumberValidator(Validator):
    def validate(self, document):
        try:
            int(document.text)
        except ValueError:
            raise ValidationError(message='Must be a number type!', cursor_position=len(document.text))

class PortValidator(Validator):
    def validate(self, document):
        try:
            port = int(document.text)
        except ValueError:
            raise ValidationError(message='Must be a number type!', cursor_position=len(document.text))

        if not 1 <= port <= 65535:
            raise ValidationError(message='Port is invalid!', cursor_position=len(document.text))

class HostValidator(NotNullValidator):
    def validate(self, document):
        super(HostValidator, self).validate(document)

        hostname = document.text
        if len(hostname) > 255:
            return ValidationError(message='Host is invalid!', cursor_position=len(document.text))

        if hostname[-1] == ".":
            hostname = hostname[:-1]

        allowed = compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", IGNORECASE)
        if not all(allowed.match(x) for x in hostname.split(".")):
            raise ValidationError(message='Host is invalid!', cursor_position=len(document.text))

class PathValidator(Validator):
    def validate(self, document):
        if not exists(document.text):
            raise ValidationError(message='File doesn\'t exist!', cursor_position=len(document.text))

        if splitext(document.text)[1] != '.pem':
            raise ValidationError(message='File must be .pem extension!', cursor_position=len(document.text))

class FileExtensionValidator(NotNullValidator):
    def validate(self, document):
        super(FileExtensionValidator, self).validate(document)

        if document.text.split('.')[-1] != 'json':
            raise ValidationError(message='File must be .json!', cursor_position=len(document.text))

def generate_config_file(data: {str: str}) -> None:
    with open(CONFIG_PATH, 'w') as file:
        dump(data, file, default_flow_style=False, sort_keys=False)

def read_config_file() -> {str: str}:
    with open(CONFIG_PATH, 'r') as file:
        config_dict = safe_load(file)

    return config_dict

def main():
    try:
        default_config = read_config_file()

        base_questions = [
            {
                'type': 'input',
                'name': 'host',
                'message': 'ThingsBoard host:',
                'default': default_config['thingsboard']['host'],
                'validate': HostValidator
            },
            {
                'type': 'input',
                'name': 'port',
                'message': 'ThingsBoard port:',
                'default': str(default_config['thingsboard']['port']),
                'validate': PortValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'confirm',
                'name': 'remoteShell',
                'message': 'Do you want to have access from remote shell? (No)',
                'default': False
            },
            {
                'type': 'confirm',
                'name': 'remoteConfiguration',
                'message': 'Do you want to enable remote configuration feature? (No)',
                'default': False
            },
            {
                'type': 'input',
                'name': 'statsSendPeriodInSeconds',
                'message': 'Statistics will be send every (sec.):',
                'default': str(default_config['thingsboard']['statsSendPeriodInSeconds']),
                'validate': NumberValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'input',
                'name': 'checkConnectorsConfigurationInSeconds',
                'message': 'Connectors config files will check every (sec.):',
                'default': str(default_config['thingsboard']['checkConnectorsConfigurationInSeconds']),
                'validate': NumberValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'list',
                'name': 'security',
                'message': 'What security type do you need?',
                'choices': [
                    'Access Token (Basic Security)',
                    'TLS + Access Token (Advanced Security)',
                    'TLS + Private Key (Advanced Security)'
                ]
            }
        ]

        f = Figlet(font='slant')
        print(colored(f.renderText('TB IoT Gateway'), color='blue'))
        print(colored('Welcome to ThingsBoard IoT Gateway Setup Wizard', 'blue'))
        print(colored('Let\'s configure you Gateway by answering on questions below ⬇\n'))

        base_answers = prompt(base_questions, style=style)

        access_token_config = [
            {
                'type': 'input',
                'name': 'accessToken',
                'message': 'Your token:',
                'validate': NotNullValidator
            }
        ]
        tls = [
            {
                'type': 'input',
                'name': 'caCert',
                'message': 'Path to your CA file (.pem):',
                'validate': PathValidator
            }
        ]
        tls_access_token_config = access_token_config + tls
        tls_private_key_config = [
                                     {
                                         'type': 'input',
                                         'name': 'privateKey',
                                         'message': 'Path to you private key file (.pem):',
                                         'validate': PathValidator
                                     }
                                 ] + tls + [
                                     {
                                         'type': 'input',
                                         'name': 'cert',
                                         'message': 'Path to your certificate file (.pem):',
                                         'validate': PathValidator
                                     }
                                 ]

        if base_answers['security'] == 'Access Token (Basic Security)':
            security_questions = access_token_config
        elif base_answers['security'] == 'TLS + Access Token (Advanced Security)':
            security_questions = tls_access_token_config
        else:
            security_questions = tls_private_key_config

        security_answers = prompt(security_questions, style=style)

        qos_and_storage_type_question = [
            {
                'type': 'input',
                'name': 'qos',
                'message': 'QoS:',
                'validate': NumberValidator,
                'default': str(default_config['thingsboard']['qos']),
                'filter': lambda val: int(val)
            },
            {
                'type': 'list',
                'name': 'storage',
                'message': 'Choose storage type:',
                'choices': [
                    'Memory',
                    'File storage'
                ],
                'filter': lambda val: val.lower() if val == 'Memory' else 'file'
            }
        ]

        qos_and_storage_type_answers = prompt(qos_and_storage_type_question, style=style)

        if qos_and_storage_type_answers['storage'] == 'memory':
            storage_questions = [
                {
                    'type': 'input',
                    'name': 'read_records_count',
                    'message': 'Count of messages to get from storage and send to ThingsBoard:',
                    'default': str(default_config['storage'].get('read_records_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'max_records_count',
                    'message': 'Maximum count of data in storage before send to ThingsBoard:',
                    'default': str(default_config['storage'].get('max_records_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                }
            ]
        else:
            storage_questions = [
                {
                    'type': 'input',
                    'name': 'data_folder_path',
                    'message': 'Path to folder, that will contains data (Relative or Absolute):',
                    'default': str(default_config['storage'].get('data_folder_path', '')),
                    'validate': NotNullValidator
                },
                {
                    'type': 'input',
                    'name': 'max_file_count',
                    'message': 'Maximum count of file that will be saved:',
                    'default': str(default_config['storage'].get('max_file_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'max_read_records_count',
                    'message': 'Count of messages to get from storage and send to ThingsBoard:',
                    'default': str(default_config['storage'].get('max_read_records_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'max_records_per_file',
                    'message': 'Maximum count of records that will be stored in one file:',
                    'default': str(default_config['storage'].get('max_records_per_file', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                }
            ]

        storage_answers = prompt(storage_questions, style=style)

        connectors_questions = [
            {
                'type': 'checkbox',
                'name': 'connectors',
                'message': 'Choose connectors you want to use:',
                'choices': [
                    {
                        'name': 'MQTT',
                    },
                    {
                        'name': 'FTP',
                    },
                    {
                        'name': 'Modbus',
                    },
                    {
                        'name': 'CAN',
                    },
                    {
                        'name': 'Bacnet',
                    },
                    {
                        'name': 'BLE',
                    },
                    {
                        'name': 'OPC-UA',
                    },
                    {
                        'name': 'ODBC',
                    },
                    {
                        'name': 'Request',
                    },
                    {
                        'name': 'REST',
                    },
                    {
                        'name': 'SNMP'
                    }
                ],
                'validate': lambda answer: 'You must choose at least one connector.' if len(answer) == 0 else True
            }
        ]

        connectors_answers = prompt(connectors_questions, style=style)

        connectors_list = []

        for connector in connectors_answers['connectors']:
            print(colored(f'Configuration {connector} connector:', 'blue'))
            connector_questions = [
                {
                    'type': 'input',
                    'name': 'name',
                    'message': 'Name of connector:',
                    'validate': NotNullValidator
                },
                {
                    'type': 'input',
                    'name': 'configuration',
                    'message': 'Config file of connector:',
                    'validate': FileExtensionValidator
                }
            ]
            connector_answers = prompt(connector_questions, style=style)
            connectors_list.append({'type': connector.lower(), **connector_answers})

        generate_config_file(
            {
                'thingsboard': {**base_answers, 'security': security_answers,
                                'qos': qos_and_storage_type_answers['qos']},
                'storage': {'type': qos_and_storage_type_answers['storage'], **storage_answers},
                'connectors': connectors_list
            })
    except Exception:
        print(colored('Something went wrong! Please try again.', color='red'))
        raise SystemExit

main()
<commit_msg>Moved call for main function<commit_after>from __future__ import print_function, unicode_literals

from os import path

from PyInquirer import style_from_dict, Token, prompt
from pyfiglet import Figlet
from termcolor import colored
from prompt_toolkit.validation import ValidationError, Validator
from re import compile, IGNORECASE
from os.path import exists, splitext
from yaml import safe_load, dump

if exists('thingsboard_gateway/config/tb_gateway.yaml'):
    CONFIG_PATH = 'thingsboard_gateway/config/tb_gateway.yaml'
elif exists(path.dirname(path.abspath(__file__)) + '/config/tb_gateway.yaml'.replace('/', path.sep)):
    CONFIG_PATH = path.dirname(path.abspath(__file__)) + '/config/tb_gateway.yaml'.replace('/', path.sep)
elif exists("/etc/thingsboard-gateway/config/tb_gateway.yaml".replace('/', path.sep)):
    CONFIG_PATH = "/etc/thingsboard-gateway/config/tb_gateway.yaml".replace('/', path.sep)

style = style_from_dict({
    Token.Separator: '#cc5454',
    Token.QuestionMark: '#673ab7 bold',
    Token.Selected: '#673ab7',
    Token.Pointer: '#673ab7 bold',
    Token.Instruction: '',  # default
    Token.Answer: '#673ab7 bold',
    Token.Question: '',
})

class NotNullValidator(Validator):
    def validate(self, document):
        if not document.text or document.text == '':
            raise ValidationError(message='Value can be empty!', cursor_position=len(document.text))

class NumberValidator(Validator):
    def validate(self, document):
        try:
            int(document.text)
        except ValueError:
            raise ValidationError(message='Must be a number type!', cursor_position=len(document.text))

class PortValidator(Validator):
    def validate(self, document):
        try:
            port = int(document.text)
        except ValueError:
            raise ValidationError(message='Must be a number type!', cursor_position=len(document.text))

        if not 1 <= port <= 65535:
            raise ValidationError(message='Port is invalid!', cursor_position=len(document.text))

class HostValidator(NotNullValidator):
    def validate(self, document):
        super(HostValidator, self).validate(document)

        hostname = document.text
        if len(hostname) > 255:
            return ValidationError(message='Host is invalid!', cursor_position=len(document.text))

        if hostname[-1] == ".":
            hostname = hostname[:-1]

        allowed = compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", IGNORECASE)
        if not all(allowed.match(x) for x in hostname.split(".")):
            raise ValidationError(message='Host is invalid!', cursor_position=len(document.text))

class PathValidator(Validator):
    def validate(self, document):
        if not exists(document.text):
            raise ValidationError(message='File doesn\'t exist!', cursor_position=len(document.text))

        if splitext(document.text)[1] != '.pem':
            raise ValidationError(message='File must be .pem extension!', cursor_position=len(document.text))

class FileExtensionValidator(NotNullValidator):
    def validate(self, document):
        super(FileExtensionValidator, self).validate(document)

        if document.text.split('.')[-1] != 'json':
            raise ValidationError(message='File must be .json!', cursor_position=len(document.text))

def generate_config_file(data: {str: str}) -> None:
    with open(CONFIG_PATH, 'w') as file:
        dump(data, file, default_flow_style=False, sort_keys=False)

def read_config_file() -> {str: str}:
    with open(CONFIG_PATH, 'r') as file:
        config_dict = safe_load(file)

    return config_dict

def main():
    try:
        default_config = read_config_file()

        base_questions = [
            {
                'type': 'input',
                'name': 'host',
                'message': 'ThingsBoard host:',
                'default': default_config['thingsboard']['host'],
                'validate': HostValidator
            },
            {
                'type': 'input',
                'name': 'port',
                'message': 'ThingsBoard port:',
                'default': str(default_config['thingsboard']['port']),
                'validate': PortValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'confirm',
                'name': 'remoteShell',
                'message': 'Do you want to have access from remote shell? (No)',
                'default': False
            },
            {
                'type': 'confirm',
                'name': 'remoteConfiguration',
                'message': 'Do you want to enable remote configuration feature? (No)',
                'default': False
            },
            {
                'type': 'input',
                'name': 'statsSendPeriodInSeconds',
                'message': 'Statistics will be send every (sec.):',
                'default': str(default_config['thingsboard']['statsSendPeriodInSeconds']),
                'validate': NumberValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'input',
                'name': 'checkConnectorsConfigurationInSeconds',
                'message': 'Connectors config files will check every (sec.):',
                'default': str(default_config['thingsboard']['checkConnectorsConfigurationInSeconds']),
                'validate': NumberValidator,
                'filter': lambda val: int(val)
            },
            {
                'type': 'list',
                'name': 'security',
                'message': 'What security type do you need?',
                'choices': [
                    'Access Token (Basic Security)',
                    'TLS + Access Token (Advanced Security)',
                    'TLS + Private Key (Advanced Security)'
                ]
            }
        ]

        f = Figlet(font='slant')
        print(colored(f.renderText('TB IoT Gateway'), color='blue'))
        print(colored('Welcome to ThingsBoard IoT Gateway Setup Wizard', 'blue'))
        print(colored('Let\'s configure you Gateway by answering on questions below ⬇\n'))

        base_answers = prompt(base_questions, style=style)

        access_token_config = [
            {
                'type': 'input',
                'name': 'accessToken',
                'message': 'Your token:',
                'validate': NotNullValidator
            }
        ]
        tls = [
            {
                'type': 'input',
                'name': 'caCert',
                'message': 'Path to your CA file (.pem):',
                'validate': PathValidator
            }
        ]
        tls_access_token_config = access_token_config + tls
        tls_private_key_config = [
                                     {
                                         'type': 'input',
                                         'name': 'privateKey',
                                         'message': 'Path to you private key file (.pem):',
                                         'validate': PathValidator
                                     }
                                 ] + tls + [
                                     {
                                         'type': 'input',
                                         'name': 'cert',
                                         'message': 'Path to your certificate file (.pem):',
                                         'validate': PathValidator
                                     }
                                 ]

        if base_answers['security'] == 'Access Token (Basic Security)':
            security_questions = access_token_config
        elif base_answers['security'] == 'TLS + Access Token (Advanced Security)':
            security_questions = tls_access_token_config
        else:
            security_questions = tls_private_key_config

        security_answers = prompt(security_questions, style=style)

        qos_and_storage_type_question = [
            {
                'type': 'input',
                'name': 'qos',
                'message': 'QoS:',
                'validate': NumberValidator,
                'default': str(default_config['thingsboard']['qos']),
                'filter': lambda val: int(val)
            },
            {
                'type': 'list',
                'name': 'storage',
                'message': 'Choose storage type:',
                'choices': [
                    'Memory',
                    'File storage'
                ],
                'filter': lambda val: val.lower() if val == 'Memory' else 'file'
            }
        ]

        qos_and_storage_type_answers = prompt(qos_and_storage_type_question, style=style)

        if qos_and_storage_type_answers['storage'] == 'memory':
            storage_questions = [
                {
                    'type': 'input',
                    'name': 'read_records_count',
                    'message': 'Count of messages to get from storage and send to ThingsBoard:',
                    'default': str(default_config['storage'].get('read_records_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'max_records_count',
                    'message': 'Maximum count of data in storage before send to ThingsBoard:',
                    'default': str(default_config['storage'].get('max_records_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                }
            ]
        else:
            storage_questions = [
                {
                    'type': 'input',
                    'name': 'data_folder_path',
                    'message': 'Path to folder, that will contains data (Relative or Absolute):',
                    'default': str(default_config['storage'].get('data_folder_path', '')),
                    'validate': NotNullValidator
                },
                {
                    'type': 'input',
                    'name': 'max_file_count',
                    'message': 'Maximum count of file that will be saved:',
                    'default': str(default_config['storage'].get('max_file_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'max_read_records_count',
                    'message': 'Count of messages to get from storage and send to ThingsBoard:',
                    'default': str(default_config['storage'].get('max_read_records_count', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                },
                {
                    'type': 'input',
                    'name': 'max_records_per_file',
                    'message': 'Maximum count of records that will be stored in one file:',
                    'default': str(default_config['storage'].get('max_records_per_file', '')),
                    'validate': NumberValidator,
                    'filter': lambda val: int(val)
                }
            ]

        storage_answers = prompt(storage_questions, style=style)

        connectors_questions = [
            {
                'type': 'checkbox',
                'name': 'connectors',
                'message': 'Choose connectors you want to use:',
                'choices': [
                    {
                        'name': 'MQTT',
                    },
                    {
                        'name': 'FTP',
                    },
                    {
                        'name': 'Modbus',
                    },
                    {
                        'name': 'CAN',
                    },
                    {
                        'name': 'Bacnet',
                    },
                    {
                        'name': 'BLE',
                    },
                    {
                        'name': 'OPC-UA',
                    },
                    {
                        'name': 'ODBC',
                    },
                    {
                        'name': 'Request',
                    },
                    {
                        'name': 'REST',
                    },
                    {
                        'name': 'SNMP'
                    }
                ],
                'validate': lambda answer: 'You must choose at least one connector.' if len(answer) == 0 else True
            }
        ]

        connectors_answers = prompt(connectors_questions, style=style)

        connectors_list = []

        for connector in connectors_answers['connectors']:
            print(colored(f'Configuration {connector} connector:', 'blue'))
            connector_questions = [
                {
                    'type': 'input',
                    'name': 'name',
                    'message': 'Name of connector:',
                    'validate': NotNullValidator
                },
                {
                    'type': 'input',
                    'name': 'configuration',
                    'message': 'Config file of connector:',
                    'validate': FileExtensionValidator
                }
            ]
            connector_answers = prompt(connector_questions, style=style)
            connectors_list.append({'type': connector.lower(), **connector_answers})

        generate_config_file(
            {
                'thingsboard': {**base_answers, 'security': security_answers,
                                'qos': qos_and_storage_type_answers['qos']},
                'storage': {'type': qos_and_storage_type_answers['storage'], **storage_answers},
                'connectors': connectors_list
            })
    except Exception:
        print(colored('Something went wrong! Please try again.', color='red'))
        raise SystemExit

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import sys
import fcntl
import errno
import atexit
import logging
import socket
import ssl
import time
import datetime
import calendar
import re
import string
if sys.version_info[0] < 3:
    from Queue import Queue, Full
else:
    from queue import Queue, Full

from threading import Thread
from logging import NullHandler
import pkg_resources

def normalize_time(time_like):
    """Returns unix timestamp integer for all common time/duration formats."""
    if isinstance(time_like, time.struct_time):
        time_like = time.mktime(time_like)
    if isinstance(time_like, datetime.datetime):
        time_like = calendar.timegm(time_like.utctimetuple())
    if isinstance(time_like, datetime.timedelta):
        time_like = time_like.total_seconds()
    return int(time_like)

def is_valid(metric, value, time, count):
    """Returns True/False if a metric/value/time/count is valid"""
    valid_metric = re.search(r"^([\d\w\-_]+\.)*[\d\w\-_]+$", metric)

    valid_value = re.search(r"^-?\d+(\.\d+)?(e-\d+)?$", str(value))

    if valid_metric and valid_value:
        return True

    # TODO
    # report_invalid_metric(metric) unless valid_metric
    # report_invalid_value(metric, value) unless valid_value
    return False

def is_valid_note(message):
    """Returns True/False if a notice message is valid."""
    return not bool(re.search("[\n\r]", message))

def join(strings, joiner):
    """
    Joins a list of strings together with an interleaved joiner string.
    This is a compatibility function for Python 2/3.
    """
    if sys.version_info[0] < 3:
        return string.join(strings, joiner)
    else:
        return joiner.join(strings)

def add_stderr_logger(level=logging.DEBUG):
    """
    Helper for quickly adding a StreamHandler to the logger. Useful for
    debugging.

    Returns the handler after adding it.
    """
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    logger.addHandler(handler)
    logger.setLevel(level)
    logger.debug('Added a stderr logging handler to logger: %s', __name__)
    return handler

class Agent(object):
    """
    Used to connect to Instrumental and send metric data.
    """
    backoff = 2
    connect_timeout = 20
    reply_timeout = 10
    exit_flush_timeout = 5
    hostname = socket.gethostname()
    max_buffer = 5000
    max_reconnect_delay = 15
    exit_timeout = 1
    version = "1.1.0"

    def __init__(self, api_key, collector="collector.instrumentalapp.com:8001", enabled=True, secure=True, verify_cert=True, synchronous=False):
        self.logger = logging.getLogger(__name__)
        self.logger.addHandler(logging.NullHandler())
        self.logger.setLevel(logging.DEBUG)
        self.logger.debug("Initializing...")

        self.api_key = api_key
        self.host, self.port = collector.split(":")
        self.port = int(self.port)
        self.secure = secure
        self.verify_cert = verify_cert
        self.enabled = enabled
        self.synchronous = synchronous
        self.worker = False
        self.bare_socket = False
        self.socket = False

        self.pid = None
        self.failures = 0

        if self.enabled:
            self.queue = Queue(Agent.max_buffer)
            self._setup_cleanup_at_exit()

    def gauge(self, metric, value, timestamp=None, count=1):
        """
        Store a gauge for a metric, optionally at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("gauge", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def increment(self, metric, value=1, timestamp=None, count=1):
        """
        Increment a metric, optionally more than one or at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("increment", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def notice(self, note, timestamp=None, duration=0):
        """
        Records a note at a specific time and duration. Useful for things like
        deploys or other significant changes.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid_note(note):
                self._send_command("notice", normalize_time(timestamp), normalize_time(duration), note)
                return note
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time(self, metric, fun, multiplier=1):
        """
        Store the execution duration of a function in a metric. Multiplier can
        be used to scale the duration to desired unit or change the duration
        in some meaningful way. Default is in seconds.
        """
        try:
            start = time.time()
            value = fun()
            finish = time.time()
            duration = finish - start
            self.gauge(metric, duration * multiplier, start)
            return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time_ms(self, metric, fun):
        """
        Store the execution duration of a function in a metric. Execution time
        is measured in milliseconds.
        """
        try:
            return self.time(metric, fun, 1000)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def is_running(self):
        """Returns True/False if the worker is running"""
        return self._same_pid() and self._worker_alive()

    def _same_pid(self):
        return bool(os.getpid() == self.pid)

    def _worker_alive(self):
        return bool(self.worker) and self.worker.is_alive()

    def _setup_cleanup_at_exit(self):
        self.logger.debug("registering exit handler")
        atexit.register(self._cleanup)

    def _cleanup(self):
        try:
            if self.is_running:
                if self.queue.empty():
                    self.logger.debug("At Exit handler, join skipped, queue empty.")
                else:
                    self.logger.debug("At Exit handler, waiting up to %0.3f seconds (count: %i) ", Agent.exit_timeout, self.queue.qsize())
                    started = time.time()
                    while (time.time() - started) < Agent.exit_timeout and not self.queue.empty():
                        time.sleep(0.05)
                    if self.queue.empty():
                        self.logger.debug("All metrics pushed.")
                    else:
                        self.logger.info("Discarding %i metrics.", self.queue.qsize())
            else:
                self.logger.debug("At Exit handler, join skipped, worker not running.")
        except Exception as error:
            self.logger.error("At Exit ERROR: " + str(error))

    def _start_connection_worker(self):
        if self.enabled:
            self.pid = os.getpid()
            self.failures = 0
            self.logger.debug("Starting thread...")

            self.worker = Thread(target=self._worker_loop)
            self.worker.setDaemon(True)  # So exit handler won't wait on this
            self.worker.start()

    def _worker_loop(self):
        self.logger.debug("worker starting...")
        while True:
            try:
                self.bare_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

                if self.secure:
                    # Rely on server to enforce secure protocol/ciphers
                    self.socket = ssl.wrap_socket(self.bare_socket)
                else:
                    self.socket = self.bare_socket

                self.socket.connect((self.host, self.port))
                self.socket.settimeout(self.reply_timeout)

                def socket_send(data):
                    if sys.version_info[0] < 3:
                        self.socket.send(data)
                    else:
                        self.socket.send(bytes(data, "ASCII"))

                socket_send("hello version python/instrumental_agent/%(version)s hostname %(hostname)s\n" % {"version": Agent.version, "hostname": Agent.hostname})
                socket_send("authenticate %s\n" % self.api_key)

                data = b""
                ok_count = 0
                receiving = True
                connected = False

                while receiving:
                    data += self.socket.recv(1024)
                    while b"\n" in data:
                        self.logger.debug("initial data: %s" % repr(data))
                        response, data = data.split(b"\n", 1)
                        self.logger.debug("  response: %s" % repr(response))
                        self.logger.debug("  data: %s" % repr(data))
                        if response == b"ok":
                            ok_count += 1
                            if ok_count >= 2:
                                self.logger.debug("auth a-ok")
                                receiving = False
                                connected = True
                                break
                        else:
                            self.logger.debug("auth failed...")
                            break
                if connected:
                    self.socket.settimeout(None)
                    while True:
                        item = self.queue.get()
                        if self._test_connection():
                            socket_send(item)
                            self.queue.task_done()
                        else:
                            self.queue.put(item, False)
                            raise Exception("socket error")
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception as error:
                self.failures += 1
                delay = min((self.failures - 1) ** Agent.backoff, Agent.max_reconnect_delay)
                time.sleep(delay)
                self._log_exception(error)
                import traceback
                traceback.print_exc(file=sys.stdout)
                sys.exit()

    def _log_exception(self, error):
        self.logger.debug("EXCEPTION %s", str(error))

    def _send_command(self, cmd, *args):
        if self.enabled:
            args = [str(arg) for arg in args]
            string_cmd = "%s %s\n" % (cmd, join(args, " "))
            if not self.is_running():
                self._start_connection_worker()
            try:
                self.queue.put(string_cmd, False)
            except Full:
                self.logger.debug("Queue full(limit %i), discarding metric", Agent.max_buffer)

    def _test_connection(self):
        s = self.bare_socket
        fcntl.fcntl(s, fcntl.F_SETFL, os.O_NONBLOCK)

        try:
            msg = s.recv(1)
            return msg == ""
        except socket.error as e:
            err = e.args[0]
            if err == errno.EAGAIN or err == errno.EWOULDBLOCK:
                return True
            else:
                self.logger.debug("Socket connection error")
                return False
<commit_msg>Remove unused imports<commit_after>import os
import sys
import fcntl
import errno
import atexit
import logging
import socket
import ssl
import time
import datetime
import calendar
import re
import string
if sys.version_info[0] < 3:
    from Queue import Queue, Full
else:
    from queue import Queue, Full

from threading import Thread

def normalize_time(time_like):
    """Returns unix timestamp integer for all common time/duration formats."""
    if isinstance(time_like, time.struct_time):
        time_like = time.mktime(time_like)
    if isinstance(time_like, datetime.datetime):
        time_like = calendar.timegm(time_like.utctimetuple())
    if isinstance(time_like, datetime.timedelta):
        time_like = time_like.total_seconds()
    return int(time_like)

def is_valid(metric, value, time, count):
    """Returns True/False if a metric/value/time/count is valid"""
    valid_metric = re.search(r"^([\d\w\-_]+\.)*[\d\w\-_]+$", metric)

    valid_value = re.search(r"^-?\d+(\.\d+)?(e-\d+)?$", str(value))

    if valid_metric and valid_value:
        return True

    # TODO
    # report_invalid_metric(metric) unless valid_metric
    # report_invalid_value(metric, value) unless valid_value
    return False

def is_valid_note(message):
    """Returns True/False if a notice message is valid."""
    return not bool(re.search("[\n\r]", message))

def join(strings, joiner):
    """
    Joins a list of strings together with an interleaved joiner string.
    This is a compatibility function for Python 2/3.
    """
    if sys.version_info[0] < 3:
        return string.join(strings, joiner)
    else:
        return joiner.join(strings)

def add_stderr_logger(level=logging.DEBUG):
    """
    Helper for quickly adding a StreamHandler to the logger. Useful for
    debugging.

    Returns the handler after adding it.
    """
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    logger.addHandler(handler)
    logger.setLevel(level)
    logger.debug('Added a stderr logging handler to logger: %s', __name__)
    return handler

class Agent(object):
    """
    Used to connect to Instrumental and send metric data.
    """
    backoff = 2
    connect_timeout = 20
    reply_timeout = 10
    exit_flush_timeout = 5
    hostname = socket.gethostname()
    max_buffer = 5000
    max_reconnect_delay = 15
    exit_timeout = 1
    version = "1.1.0"

    def __init__(self, api_key, collector="collector.instrumentalapp.com:8001", enabled=True, secure=True, verify_cert=True, synchronous=False):
        self.logger = logging.getLogger(__name__)
        self.logger.addHandler(logging.NullHandler())
        self.logger.setLevel(logging.DEBUG)
        self.logger.debug("Initializing...")

        self.api_key = api_key
        self.host, self.port = collector.split(":")
        self.port = int(self.port)
        self.secure = secure
        self.verify_cert = verify_cert
        self.enabled = enabled
        self.synchronous = synchronous
        self.worker = False
        self.bare_socket = False
        self.socket = False

        self.pid = None
        self.failures = 0

        if self.enabled:
            self.queue = Queue(Agent.max_buffer)
            self._setup_cleanup_at_exit()

    def gauge(self, metric, value, timestamp=None, count=1):
        """
        Store a gauge for a metric, optionally at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("gauge", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def increment(self, metric, value=1, timestamp=None, count=1):
        """
        Increment a metric, optionally more than one or at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("increment", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def notice(self, note, timestamp=None, duration=0):
        """
        Records a note at a specific time and duration. Useful for things like
        deploys or other significant changes.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid_note(note):
                self._send_command("notice", normalize_time(timestamp), normalize_time(duration), note)
                return note
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time(self, metric, fun, multiplier=1):
        """
        Store the execution duration of a function in a metric. Multiplier can
        be used to scale the duration to desired unit or change the duration
        in some meaningful way. Default is in seconds.
        """
        try:
            start = time.time()
            value = fun()
            finish = time.time()
            duration = finish - start
            self.gauge(metric, duration * multiplier, start)
            return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time_ms(self, metric, fun):
        """
        Store the execution duration of a function in a metric. Execution time
        is measured in milliseconds.
        """
        try:
            return self.time(metric, fun, 1000)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def is_running(self):
        """Returns True/False if the worker is running"""
        return self._same_pid() and self._worker_alive()

    def _same_pid(self):
        return bool(os.getpid() == self.pid)

    def _worker_alive(self):
        return bool(self.worker) and self.worker.is_alive()

    def _setup_cleanup_at_exit(self):
        self.logger.debug("registering exit handler")
        atexit.register(self._cleanup)

    def _cleanup(self):
        try:
            if self.is_running:
                if self.queue.empty():
                    self.logger.debug("At Exit handler, join skipped, queue empty.")
                else:
                    self.logger.debug("At Exit handler, waiting up to %0.3f seconds (count: %i) ", Agent.exit_timeout, self.queue.qsize())
                    started = time.time()
                    while (time.time() - started) < Agent.exit_timeout and not self.queue.empty():
                        time.sleep(0.05)
                    if self.queue.empty():
                        self.logger.debug("All metrics pushed.")
                    else:
                        self.logger.info("Discarding %i metrics.", self.queue.qsize())
            else:
                self.logger.debug("At Exit handler, join skipped, worker not running.")
        except Exception as error:
            self.logger.error("At Exit ERROR: " + str(error))

    def _start_connection_worker(self):
        if self.enabled:
            self.pid = os.getpid()
            self.failures = 0
            self.logger.debug("Starting thread...")

            self.worker = Thread(target=self._worker_loop)
            self.worker.setDaemon(True)  # So exit handler won't wait on this
            self.worker.start()

    def _worker_loop(self):
        self.logger.debug("worker starting...")
        while True:
            try:
                self.bare_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

                if self.secure:
                    # Rely on server to enforce secure protocol/ciphers
                    self.socket = ssl.wrap_socket(self.bare_socket)
                else:
                    self.socket = self.bare_socket

                self.socket.connect((self.host, self.port))
                self.socket.settimeout(self.reply_timeout)

                def socket_send(data):
                    if sys.version_info[0] < 3:
                        self.socket.send(data)
                    else:
                        self.socket.send(bytes(data, "ASCII"))

                socket_send("hello version python/instrumental_agent/%(version)s hostname %(hostname)s\n" % {"version": Agent.version, "hostname": Agent.hostname})
                socket_send("authenticate %s\n" % self.api_key)

                data = b""
                ok_count = 0
                receiving = True
                connected = False

                while receiving:
                    data += self.socket.recv(1024)
                    while b"\n" in data:
                        self.logger.debug("initial data: %s" % repr(data))
                        response, data = data.split(b"\n", 1)
                        self.logger.debug("  response: %s" % repr(response))
                        self.logger.debug("  data: %s" % repr(data))
                        if response == b"ok":
                            ok_count += 1
                            if ok_count >= 2:
                                self.logger.debug("auth a-ok")
                                receiving = False
                                connected = True
                                break
                        else:
                            self.logger.debug("auth failed...")
                            break
                if connected:
                    self.socket.settimeout(None)
                    while True:
                        item = self.queue.get()
                        if self._test_connection():
                            socket_send(item)
                            self.queue.task_done()
                        else:
                            self.queue.put(item, False)
                            raise Exception("socket error")
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception as error:
                self.failures += 1
                delay = min((self.failures - 1) ** Agent.backoff, Agent.max_reconnect_delay)
                time.sleep(delay)
                self._log_exception(error)
                import traceback
                traceback.print_exc(file=sys.stdout)
                sys.exit()

    def _log_exception(self, error):
        self.logger.debug("EXCEPTION %s", str(error))

    def _send_command(self, cmd, *args):
        if self.enabled:
            args = [str(arg) for arg in args]
            string_cmd = "%s %s\n" % (cmd, join(args, " "))
            if not self.is_running():
                self._start_connection_worker()
            try:
                self.queue.put(string_cmd, False)
            except Full:
                self.logger.debug("Queue full(limit %i), discarding metric", Agent.max_buffer)

    def _test_connection(self):
        s = self.bare_socket
        fcntl.fcntl(s, fcntl.F_SETFL, os.O_NONBLOCK)

        try:
            msg = s.recv(1)
            return msg == ""
        except socket.error as e:
            err = e.args[0]
            if err == errno.EAGAIN or err == errno.EWOULDBLOCK:
                return True
            else:
                self.logger.debug("Socket connection error")
                return False
<|endoftext|>


<commit_before>from collections import Counter
from common.funcfun import lmap
from functools import reduce
from operator import add
from preprocess.words import get_words
from preprocess.html_remover import HTMLRemover
from common.string import normalize_text, strip_accents
from common.czech_stemmer import cz_stem
from html.parser import HTMLParseError

def getinfo(documents):
	counters = list(enumerate(map(lambda x: Counter(x), documents)))
	allwords = reduce(add, documents)
	allwords_counter = Counter(allwords)
	words = sorted(set(allwords))
	occurencesIndex = lmap(lambda x: ((x, allwords_counter[x]), occurences(counters, x)), words)
	print(allwords_counter)
	#return occurencesIndex
	return {'allwords' : allwords_counter, 'occurences' : occurencesIndex}

def occurences(counters, word):
	return lmap(lambda x: (x[0], x[1][word]), filter(lambda x: word in x[1], counters))

def group(database, keylen):
	dic = {}
	for record in database:
		key = record[0][0][:keylen]
		if key in dic:
			dic[key].append(record)
		else:
			dic[key] = [record]
	return dic

def toindex(sites, urls, keylen):
	htmlrem = HTMLRemover()
	parsedSites = []
	correctUrl = []

	for site, url in zip(sites, urls):
		try:
			parsedSites.append(get_words(normalize_text(htmlrem.compile(site)), []))
			correctUrl.append(url)
		except HTMLParseError:
			print('Cannot parse ' + str(url))

	sitesInfo = getinfo(list(parsedSites))
	index = group(sitesInfo['occurences'], keylen)

	return {'index': index, 'urls': correctUrl, 'allwords' : sitesInfo['allwords']}

def getstem(word):
	word = normalize_text(word)
	stem = cz_stem(word)
	stem = strip_accents(stem)
	return stem<commit_msg>refactoring<commit_after>from collections import Counter
from common.funcfun import lmap
from functools import reduce
from operator import add
from preprocess.words import get_words
from preprocess.html_remover import HTMLRemover
from common.string import normalize_text, strip_accents
from common.czech_stemmer import cz_stem
from html.parser import HTMLParseError

def getinfo(documents):
	counters = list(enumerate(map(lambda x: Counter(x), documents)))
	allwords = reduce(add, documents)
	allwords_counter = Counter(allwords)
	words = sorted(set(allwords))
	occurencesIndex = lmap(lambda x: ((x, allwords_counter[x]), occurences(counters, x)), words)
	print(allwords_counter)
	#return occurencesIndex
	return {'allwords' : allwords_counter, 'occurences' : occurencesIndex}

def occurences(counters, word):
	return lmap(lambda x: (x[0], x[1][word]), filter(lambda x: word in x[1], counters))

def group(database, keylen):
	dic = {}
	for record in database:
		key = record[0][0][:keylen]
		if key in dic:
			dic[key].append(record)
		else:
			dic[key] = [record]
	return dic

def toindex(sites, urls, keylen):
	htmlrem = HTMLRemover()
	parsedSites = []
	correctUrl = []

	for site, url in zip(sites, urls):
		try:
			parsedSites.append(get_words(normalize_text(htmlrem.compile(site)), []))
			correctUrl.append(url)
		except HTMLParseError:
			print('Cannot parse ' + str(url))

	sitesInfo = getinfo(list(parsedSites))
	index = group(sitesInfo['occurences'], keylen)

	return {'index': index, 'urls': correctUrl, 'allwords':sitesInfo['allwords']}

def getstem(word):
	word = normalize_text(word)
	stem = cz_stem(word)
	stem = strip_accents(stem)
	return stem<|endoftext|>