<commit_before>from .. import E

class Dimension(object):
    """
    Defines the dimension used for quantity units
    """

    element_name = 'Dimension'
    valid_dims = ['m', 'l', 't', 'i', 'n', 'k', 'j']

    def __init__(self, name, **kwargs):
        self.name = name
        for k in kwargs:
            if k not in self.valid_dims:
                raise Exception("'{}' is not a valid dimension name ('{}')"
                                .format(k, "', '".join(self.valid_dims)))
        self._dims = kwargs

    def power(self, dim_name):
        return self._dims.get(dim_name, 0)

    def to_xml(self):
        kwargs = {'name': self.name}
        kwargs.update(self._dims)
        return E(self.element_name,
                 **kwargs)

    @classmethod
    def from_xml(cls, element, _):
        kwargs = dict(element.attrib)
        name = kwargs.pop('name')
        return cls(name, **kwargs)

class Unit(object):
    """
    Defines the units of a quantity
    """

    element_name = 'Dimension'
    valid_dims = ['m', 'l', 't', 'i', 'n', 'k', 'j']

    def __init__(self, name, dimension, power, offset=0.0):
        self.name = name
        self.dimension = dimension
        self.power = power
        self.offset = offset

    @property
    def symbol(self):
        return self.name

    def to_xml(self):
        kwargs = {'symbol': self.symbol, 'dimension': self.dimension,
                  'power': self.power}
        if self.offset:
            kwargs['offset'] = self.offset
        return E(self.element_name,
                 **kwargs)

    @classmethod
    def from_xml(cls, element, context):
        name = element.attrib['symbol']
        dimension = context[element.attrib['dimension']]
        power = element.attrib['power']
        offset = element.attrib.get('name', 0.0)
        return cls(name, dimension, power, offset)
<commit_msg>Added built-in dimensions and units to Python library<commit_after>from .. import E

class Dimension(object):
    """
    Defines the dimension used for quantity units
    """

    element_name = 'Dimension'
    valid_dims = ['m', 'l', 't', 'i', 'n', 'k', 'j']

    def __init__(self, name, **kwargs):
        self.name = name
        for k in kwargs:
            if k not in self.valid_dims:
                raise Exception("'{}' is not a valid dimension name ('{}')"
                                .format(k, "', '".join(self.valid_dims)))
        self._dims = kwargs

    def __repr__(self):
        return "Dimension({})".format(self.name)

    def power(self, dim_name):
        return self._dims.get(dim_name, 0)

    def to_xml(self):
        kwargs = {'name': self.name}
        kwargs.update(dict((k, str(v)) for k, v in self._dims.items()))
        return E(self.element_name,
                 **kwargs)

    @classmethod
    def from_xml(cls, element, _):
        kwargs = dict(element.attrib)
        name = kwargs.pop('name')
        kwargs = dict((k, int(v)) for k, v in kwargs.items())
        return cls(name, **kwargs)

class Unit(object):
    """
    Defines the units of a quantity
    """

    element_name = 'Dimension'
    valid_dims = ['m', 'l', 't', 'i', 'n', 'k', 'j']

    def __init__(self, name, dimension, power, offset=0.0):
        self.name = name
        self.dimension = dimension
        self.power = power
        self.offset = offset

    @property
    def symbol(self):
        return self.name

    def to_xml(self):
        kwargs = {'symbol': self.name, 'dimension': self.dimension,
                  'power': self.power}
        if self.offset:
            kwargs['offset'] = self.offset
        return E(self.element_name,
                 **kwargs)

    @classmethod
    def from_xml(cls, element, context):
        name = element.attrib['symbol']
        dimension = context[element.attrib['dimension']]
        power = element.attrib['power']
        offset = element.attrib.get('name', 0.0)
        return cls(name, dimension, power, offset)

time = Dimension(name="time", t=1)
per_time = Dimension(name="per_time", t=-1)
voltage = Dimension(name="voltage", m=1, l=2, t=-3, i=-1)
per_voltage = Dimension(name="per_voltage", m=-1, l=-2, t=3, i=1)
conductance = Dimension(name="conductance", m=-1, l=-2, t=3, i=2)
conductanceDensity = Dimension(name="conductanceDensity", m=-1, l=-4, t=3, i=2)
capacitance = Dimension(name="capacitance", m=-1, l=-2, t=4, i=2)
specificCapacitance = Dimension(name="specificCapacitance", m=-1, l=-4, t=4, i=2)
resistance = Dimension(name="resistance", m=1, l=2, t=-3, i=-2)
resistivity = Dimension(name="resistivity", m=2, l=2, t=-3, i=-2)
charge = Dimension(name="charge", i=1, t=1)
charge_per_mole = Dimension(name="charge_per_mole", i=1, t=1, n=-1)
current = Dimension(name="current", i=1)
currentDensity = Dimension(name="currentDensity", i=1, l=-2)
length = Dimension(name="length", l=1)
area = Dimension(name="area", l=2)
volume = Dimension(name="volume", l=3)
concentration = Dimension(name="concentration", l=-3, n=1)
substance = Dimension(name="substance", n=1)
permeability = Dimension(name="permeability", l=1, t=-1)
temperature = Dimension(name="temperature", k=1)
idealGasConstantDims = Dimension(name="idealGasConstantDims", m=1, l=2, t=-2, k=-1, n=-1)
rho_factor = Dimension(name="rho_factor", l=-1, n=1, i=-1, t=-1)
dimensionless = None

s = Unit(name="s", dimension=time, power=0)
per_s = Unit(name="per_s", dimension=per_time, power=0)
Hz = Unit(name="Hz", dimension=per_time, power=0)
ms = Unit(name="ms", dimension=time, power=-3)
per_ms = Unit(name="per_ms", dimension=per_time, power=3)
m = Unit(name="m", dimension=length, power=0)
cm = Unit(name="cm", dimension=length, power=-2)
um = Unit(name="um", dimension=length, power=-6)
m2 = Unit(name="m2", dimension=area, power=0)
cm2 = Unit(name="cm2", dimension=area, power=-4)
um2 = Unit(name="um2", dimension=area, power=-12)
m3 = Unit(name="m3", dimension=volume, power=0)
cm3 = Unit(name="cm3", dimension=volume, power=-6)
litre = Unit(name="litre", dimension=volume, power=-3)
um3 = Unit(name="um3", dimension=volume, power=-18)
V = Unit(name="V", dimension=voltage, power=0)
mV = Unit(name="mV", dimension=voltage, power=-3)
per_V = Unit(name="per_V", dimension=per_voltage, power=0)
per_mV = Unit(name="per_mV", dimension=per_voltage, power=3)
ohm = Unit(name="ohm", dimension=resistance, power=0)
kohm = Unit(name="kohm", dimension=resistance, power=3)
Mohm = Unit(name="Mohm", dimension=resistance, power=6)
S = Unit(name="S", dimension=conductance, power=0)
mS = Unit(name="mS", dimension=conductance, power=-3)
uS = Unit(name="uS", dimension=conductance, power=-6)
nS = Unit(name="nS", dimension=conductance, power=-9)
pS = Unit(name="pS", dimension=conductance, power=-12)
S_per_m2 = Unit(name="S_per_m2", dimension=conductanceDensity, power=0)
mS_per_cm2 = Unit(name="mS_per_cm2", dimension=conductanceDensity, power=1)
S_per_cm2 = Unit(name="S_per_cm2", dimension=conductanceDensity, power=4)
F = Unit(name="F", dimension=capacitance, power=0)
uF = Unit(name="uF", dimension=capacitance, power=-6)
nF = Unit(name="nF", dimension=capacitance, power=-9)
pF = Unit(name="pF", dimension=capacitance, power=-12)
F_per_m2 = Unit(name="F_per_m2", dimension=specificCapacitance, power=0)
uF_per_cm2 = Unit(name="uF_per_cm2", dimension=specificCapacitance, power=-2)
ohm_m = Unit(name="ohm_m", dimension=resistivity, power=0)
kohm_cm = Unit(name="kohm_cm", dimension=resistivity, power=1)
ohm_cm = Unit(name="ohm_cm", dimension=resistivity, power=-2)
C = Unit(name="C", dimension=charge, power=0)
C_per_mol = Unit(name="C_per_mol", dimension=charge_per_mole, power=0)
A = Unit(name="A", dimension=current, power=0)
uA = Unit(name="uA", dimension=current, power=-6)
nA = Unit(name="nA", dimension=current, power=-9)
pA = Unit(name="pA", dimension=current, power=-12)
A_per_m2 = Unit(name="A_per_m2", dimension=currentDensity, power=0)
uA_per_cm2 = Unit(name="uA_per_cm2", dimension=currentDensity, power=-2)
mA_per_cm2 = Unit(name="mA_per_cm2", dimension=currentDensity, power=1)
mol_per_m3 = Unit(name="mol_per_m3", dimension=concentration, power=0)
mol_per_cm3 = Unit(name="mol_per_cm3", dimension=concentration, power=6)
M = Unit(name="M", dimension=concentration, power=3)
mM = Unit(name="mM", dimension=concentration, power=0)
mol = Unit(name="mol", dimension=substance, power=0)
m_per_s = Unit(name="m_per_s", dimension=permeability, power=0)
cm_per_s = Unit(name="cm_per_s", dimension=permeability, power=-2)
um_per_ms = Unit(name="um_per_ms", dimension=permeability, power=-3)
cm_per_ms = Unit(name="cm_per_ms", dimension=permeability, power=1)
degC = Unit(name="degC", dimension=temperature, power=0, offset=273.15)
K = Unit(name="K", dimension=temperature, power=0)
J_per_K_per_mol = Unit(name="J_per_K_per_mol", dimension=idealGasConstantDims, power=0)
mol_per_m_per_A_per_s = Unit(name="mol_per_m_per_A_per_s", dimension=rho_factor, power=0)<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import collections
import datetime
import logging
import sqlite3

class Error(Exception):
    pass

class _SQLObjectMixin(object):
    @classmethod
    def _columns(cls):
        return ", ".join(cls._fields)

    @classmethod
    def _qualified_columns(cls):
        return ", ".join("%s.%s" % (cls._table, f) for f in cls._fields)

    @classmethod
    def _vars(cls):
        return ", ".join("?" * len(cls._fields))

    @classmethod
    def _columns_nokey(cls):
        return ", ".join(cls._fields[1:])

    @classmethod
    def _vars_nokey(cls):
        return ", ".join("?" * (len(cls._fields) - 1))

    def _nokey(self):
        return self[1:]

class Revent(
        collections.namedtuple("ReventBase", [
            "version", "created", "author", "log"]),
        _SQLObjectMixin):
    _table = "revision"

class Dirent(
        collections.namedtuple("DirentBase", [
            "rowid", "abspath", "version", "deleted",
            "kind", "origin", "copied", "subtree"]),
        _SQLObjectMixin):
    _table = "dirindex"

    def __str__(self):
        return "%3d %c%c %s" % (
            self.version,
            self.deleted and "x" or " ",
            self.kind and "f" or "d",
            self.abspath)

class Index(object):
    def __init__(self, database):
        self.conn = sqlite3.connect(database, isolation_level = "IMMEDIATE")
        self.cursor = self.conn.cursor()
        self.cursor.execute("PRAGMA foreign_keys = ON")

    def execute(self, statement, parameters=None):
        if parameters is not None:
            fmt = statement.replace("%", "%%").replace("?", "%r")
            logging.log(logging.DEBUG//2, "EXECUTE: " + fmt, *parameters)
            return self.cursor.execute(statement, parameters)
        else:
            logging.log(logging.DEBUG//2, "EXECUTE: %s", statement)
            return self.cursor.execute(statement)

    __initialize = (
        """\
DROP TABLE IF EXISTS dirindex""",
        """\
DROP TABLE IF EXISTS revision""",
        """\
CREATE TABLE revision (
  version integer NOT NULL PRIMARY KEY,
  created timestamp NOT NULL,
  author  varchar NULL,
  log     varchar NULL
)""",
        """\
CREATE TABLE dirindex (
  rowid   integer PRIMARY KEY,
  abspath varchar NOT NULL,
  version integer NOT NULL REFERENCES revision(version),
  deleted boolean NOT NULL,
  kind    integer NOT NULL,
  origin  integer NULL REFERENCES dirindex(rowid),
  copied  boolean NOT NULL,
  subtree boolean NOT NULL
)""",
        """\
CREATE UNIQUE INDEX dirindex_versioned_tree
  ON dirindex(abspath ASC, version DESC)""",
        """\
CREATE INDEX dirindex_successor_list
  ON dirindex(origin)""",
        """\
INSERT INTO revision (version, created, author, log)
  VALUES (0, 'EPOCH', NULL, NULL)""",
        """\
INSERT INTO dirindex (abspath, version, deleted, kind, origin, copied, subtree)
  VALUES ('/', 0, 0, 0, NULL, 0, 0)""")
    def initialize(self):
        try:
            for statement in self.__initialize:
                self.execute(statement)
            self.commit()
        finally:
            self.rollback()

    def commit(self):
        logging.log(logging.DEBUG//2, "COMMIT")
        return self.conn.commit()

    def rollback(self):
        logging.log(logging.DEBUG//2, "ROLLBACK")
        return self.conn.rollback()

    def close(self):
        self.rollback()
        logging.log(logging.DEBUG//2, "CLOSE")
        return self.conn.close()

    __get_revision = """\
SELECT %s FROM revision
WHERE version = ?""" % Revent._qualified_columns()
    def get_revision(self, version):
        self.execute(self.__get_revision, [version])
        row = self.cursor.fetchone()
        if row is not None:
            return Revent._make(row)
        return None

    __new_revision = """\
INSERT INTO revision (%s)
  VALUES (%s)""" % (Revent._columns(), Revent._vars())
    def new_revision(self, version, created=None, author=None, log=None):
        if created is None:
            created = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
        revent = Revent(version = version,
                        created = created,
                        author = author,
                        log = log)
        self.execute(self.__new_revision, revent)
        return revent

    __insert = """\
INSERT INTO dirindex (%s)
  VALUES (%s)""" % (
    Dirent._columns_nokey(), Dirent._vars_nokey())
    def insert(self, dirent):
        assert isinstance(dirent, Dirent)
        self.execute(self.__insert, dirent._nokey())
        return Dirent(self.cursor.lastrowid, *dirent._nokey())

    __lookup = """\
SELECT %s FROM dirindex
WHERE abspath = ? AND version <= ?
ORDER BY abspath ASC, version DESC
LIMIT 1""" % Dirent._qualified_columns()
    def lookup(self, abspath, version):
        self.execute(self.__lookup, [abspath, version])
        row = self.cursor.fetchone()
        if row is not None:
            dirent = Dirent._make(row)
            if not dirent.deleted:
                return dirent
        return None

    __subtree = """\
SELECT %s FROM dirindex
  JOIN (SELECT abspath, MAX(version) AS maxver FROM dirindex
        WHERE version <= ? GROUP BY abspath) AS filtered
  ON dirindex.abspath == filtered.abspath
     AND dirindex.version == filtered.maxver
WHERE dirindex.abspath LIKE ? ESCAPE '#'
      AND NOT dirindex.deleted
ORDER BY dirindex.abspath ASC""" % Dirent._qualified_columns()
    def subtree(self, abspath, version):
        pattern = (abspath.rstrip("/")
                   .replace("#", "##")
                   .replace("%", "#%")
                   .replace("_", "#_")) + "/%"
        self.execute(self.__subtree, [version, pattern])
        for row in self.cursor:
            yield Dirent._make(row)

    __predecessor = """\
SELECT %s FROM dirindex
WHERE rowid = ?"""  % Dirent._qualified_columns()
    def predecessor(self, dirent):
        assert isinstance(dirent, Dirent)
        if dirent.origin is None:
            return None
        self.execute(self.__predecessor, [dirent.origin])
        return Dirent._make(self.cursor.fetchone())

    __successors = """\
SELECT %s FROM dirindex
where origin = ?
ORDER BY abspath ASC, version ASC""" % Dirent._qualified_columns()
    def successors(self, dirent):
        assert isinstance(dirent, Dirent)
        self.execute(self.__successors, [dirent.rowid])
        for row in self.cursor:
            yield Dirent._make(row)

class Revision(object):
    def __init__(self, index, version,
                 created=None, author=None, log=None):
        self.index = index
        self.version = version
        self.revent = index.get_revision(version)
        self.__created = created
        self.__author = author
        self.__log = log
        self.__context = None
        index.rollback()

    def __enter__(self):
        if self.revent is not None:
            raise Error("revision is read-only")
        logging.log(logging.DEBUG//2, "BEGIN")
        self.revent = self.index.new_revision(
            self.version, self.__created, self.__author, self.__log)
        self.__context = {}
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            if exc_type is None and len(self.__context):
                for dirent in sorted(self.__context.itervalues()):
                    self.index.insert(dirent)
                self.index.commit()
        except:
            self.index.rollback()
            raise
        finally:
            self.__context = None

    def __check_writable(self, action):
        if self.__context is None:
            raise Error(action + " requires a transaction")

    def __check_not_root(self, abspath, action):
        if abspath.rstrip("/") == "":
            raise Error(action + " not allowed on /")

    def __find_target(self, abspath, action):
        target = self.index.lookup(abspath, self.version - 1)
        if target is None:
            raise Error(action + " target does not exist: " + abspath)
        return target

    def lookup(self, abspath):
        try:
            return self.index.lookup(abspath, self.version)
        finally:
            if self.__context is None:
                self.index.rollback()

    def __add(self, action, abspath, kind, frompath, fromver):
        origin = None
        if frompath is not None:
            origin = self.index.lookup(frompath, fromver)
            if origin is None:
                raise Error(action + " source does not exist: " + frompath)
            if origin.kind != kind:
                raise Error(action + " changes the source object kind")
            origin = origin.rowid
        dirent = Dirent(rowid = None,
                        abspath = abspath,
                        version = self.version,
                        deleted = 0,
                        kind = kind,
                        origin = origin,
                        copied = int(origin is not None),
                        subtree = 0)
        self.__context[dirent.abspath] = dirent
        if frompath is not None:
            offset = len(frompath.rstrip("/"))
            prefix = abspath.rstrip("/")
            for source in self.index.subtree(frompath, fromver):
                dirent = Dirent(rowid = None,
                                abspath = prefix + source.abspath[offset:],
                                version = self.version,
                                deleted = 0,
                                kind = source.kind,
                                origin = source.rowid,
                                copied = 1,
                                subtree = 1)
                self.__context[dirent.abspath] = dirent

    def add(self, abspath, kind, frompath=None, fromver=None):
        self.__check_writable("add")
        self.__check_not_root(abspath, "add")
        return self.__add("add", abspath, kind, frompath, fromver)

    def replace(self, abspath, kind, frompath=None, fromver=None):
        self.__check_writable("replace")
        self.__check_not_root(abspath, "replace")
        self.__find_target(abspath, "replace")
        return self.__add("replace", abspath, kind, frompath, fromver)

    def modify(self, abspath):
        self.__check_writable("modify")
        target = self.__find_target(abspath, "modify")
        dirent = Dirent(rowid = None,
                        abspath = abspath,
                        version = self.version,
                        deleted = 0,
                        kind = target.kind,
                        origin = target.rowid,
                        copied = 0,
                        subtree = 0)
        self.__context[dirent.abspath] = dirent

    def delete(self, abspath):
        self.__check_writable("delete")
        self.__check_not_root(abspath, "delete")
        target = self.__find_target(abspath, "delete")
        dirent = Dirent(rowid = None,
                        abspath = abspath,
                        version = self.version,
                        deleted = 1,
                        kind = target.kind,
                        origin = target.rowid,
                        copied = 0,
                        subtree = 0)
        self.__context[dirent.abspath] = dirent
        for source in self.index.subtree(abspath, self.version - 1):
            dirent = Dirent(rowid = None,
                            abspath = source.abspath,
                            version = self.version,
                            deleted = 1,
                            kind = source.kind,
                            origin = source.rowid,
                            copied = 0,
                            subtree = 1)
            self.__context[dirent.abspath] = dirent

def simpletest(database):
    ix = Index(database)
    ix.initialize()
    with Revision(ix, 1) as rev:
        rev.add(u'/A', 0)
        rev.add(u'/A/B', 0)
        rev.add(u'/A/B/c', 1)
    with Revision(ix, 2) as rev:
        rev.add(u'/A/B/d', 1)
    with Revision(ix, 3) as rev:
        rev.add(u'/X', 0, u'/A', 1)
        rev.add(u'/X/B/d', 1, u'/A/B/d', 2)
    with Revision(ix, 4) as rev:
        rev.delete(u'/X/B/d')
        rev.add(u'/X/B/x', 1, u'/X/B/d', 3)
    with Revision(ix, 5) as rev:
        rev.delete(u'/A')

    for r in (0, 1, 2, 3, 4, 5):
        print "Revision: %d" % r
        for dirent in list(ix.subtree('/', r)):
            origin = ix.predecessor(dirent)
            if origin is None:
                print "   " + str(dirent)
            else:
                print "   %-17s  <- %s" % (dirent, origin)

    dirent = ix.lookup('/A/B/c', 4)
    print "/A/B/c@4 -> %s@%d" % (dirent.abspath, dirent.version)
    for succ in ix.successors(dirent):
        print "%11s %s %s@%d" % (
            "", succ.deleted and "x_x" or "-->",
            succ.abspath, succ.version)

    ix.close()

def loggedsimpletest(database):
    import sys
    logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
    simpletest(database)
<commit_msg>Find operation targets amongst entries in the currently open transaction as well as those already committed.<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import collections
import datetime
import logging
import sqlite3

class Error(Exception):
    pass

class _SQLObjectMixin(object):
    @classmethod
    def _columns(cls):
        return ", ".join(cls._fields)

    @classmethod
    def _qualified_columns(cls):
        return ", ".join("%s.%s" % (cls._table, f) for f in cls._fields)

    @classmethod
    def _vars(cls):
        return ", ".join("?" * len(cls._fields))

    @classmethod
    def _columns_nokey(cls):
        return ", ".join(cls._fields[1:])

    @classmethod
    def _vars_nokey(cls):
        return ", ".join("?" * (len(cls._fields) - 1))

    def _nokey(self):
        return self[1:]

class Revent(
        collections.namedtuple("ReventBase", [
            "version", "created", "author", "log"]),
        _SQLObjectMixin):
    _table = "revision"

class Dirent(
        collections.namedtuple("DirentBase", [
            "rowid", "abspath", "version", "deleted",
            "kind", "origin", "copied", "subtree"]),
        _SQLObjectMixin):
    _table = "dirindex"

    def __str__(self):
        return "%3d %c%c %s" % (
            self.version,
            self.deleted and "x" or " ",
            self.kind and "f" or "d",
            self.abspath)

class Index(object):
    def __init__(self, database):
        self.conn = sqlite3.connect(database, isolation_level = "IMMEDIATE")
        self.cursor = self.conn.cursor()
        self.cursor.execute("PRAGMA foreign_keys = ON")

    def execute(self, statement, parameters=None):
        if parameters is not None:
            fmt = statement.replace("%", "%%").replace("?", "%r")
            logging.log(logging.DEBUG//2, "EXECUTE: " + fmt, *parameters)
            return self.cursor.execute(statement, parameters)
        else:
            logging.log(logging.DEBUG//2, "EXECUTE: %s", statement)
            return self.cursor.execute(statement)

    __initialize = (
        """\
DROP TABLE IF EXISTS dirindex""",
        """\
DROP TABLE IF EXISTS revision""",
        """\
CREATE TABLE revision (
  version integer NOT NULL PRIMARY KEY,
  created timestamp NOT NULL,
  author  varchar NULL,
  log     varchar NULL
)""",
        """\
CREATE TABLE dirindex (
  rowid   integer PRIMARY KEY,
  abspath varchar NOT NULL,
  version integer NOT NULL REFERENCES revision(version),
  deleted boolean NOT NULL,
  kind    integer NOT NULL,
  origin  integer NULL REFERENCES dirindex(rowid),
  copied  boolean NOT NULL,
  subtree boolean NOT NULL
)""",
        """\
CREATE UNIQUE INDEX dirindex_versioned_tree
  ON dirindex(abspath ASC, version DESC)""",
        """\
CREATE INDEX dirindex_successor_list
  ON dirindex(origin)""",
        """\
INSERT INTO revision (version, created, author, log)
  VALUES (0, 'EPOCH', NULL, NULL)""",
        """\
INSERT INTO dirindex (abspath, version, deleted, kind, origin, copied, subtree)
  VALUES ('/', 0, 0, 0, NULL, 0, 0)""")
    def initialize(self):
        try:
            for statement in self.__initialize:
                self.execute(statement)
            self.commit()
        finally:
            self.rollback()

    def commit(self):
        logging.log(logging.DEBUG//2, "COMMIT")
        return self.conn.commit()

    def rollback(self):
        logging.log(logging.DEBUG//2, "ROLLBACK")
        return self.conn.rollback()

    def close(self):
        self.rollback()
        logging.log(logging.DEBUG//2, "CLOSE")
        return self.conn.close()

    __get_revision = """\
SELECT %s FROM revision
WHERE version = ?""" % Revent._qualified_columns()
    def get_revision(self, version):
        self.execute(self.__get_revision, [version])
        row = self.cursor.fetchone()
        if row is not None:
            return Revent._make(row)
        return None

    __new_revision = """\
INSERT INTO revision (%s)
  VALUES (%s)""" % (Revent._columns(), Revent._vars())
    def new_revision(self, version, created=None, author=None, log=None):
        if created is None:
            created = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
        revent = Revent(version = version,
                        created = created,
                        author = author,
                        log = log)
        self.execute(self.__new_revision, revent)
        return revent

    __insert = """\
INSERT INTO dirindex (%s)
  VALUES (%s)""" % (
    Dirent._columns_nokey(), Dirent._vars_nokey())
    def insert(self, dirent):
        assert isinstance(dirent, Dirent)
        self.execute(self.__insert, dirent._nokey())
        return Dirent(self.cursor.lastrowid, *dirent._nokey())

    __lookup = """\
SELECT %s FROM dirindex
WHERE abspath = ? AND version <= ?
ORDER BY abspath ASC, version DESC
LIMIT 1""" % Dirent._qualified_columns()
    def lookup(self, abspath, version):
        self.execute(self.__lookup, [abspath, version])
        row = self.cursor.fetchone()
        if row is not None:
            dirent = Dirent._make(row)
            if not dirent.deleted:
                return dirent
        return None

    __subtree = """\
SELECT %s FROM dirindex
  JOIN (SELECT abspath, MAX(version) AS maxver FROM dirindex
        WHERE version <= ? GROUP BY abspath) AS filtered
  ON dirindex.abspath == filtered.abspath
     AND dirindex.version == filtered.maxver
WHERE dirindex.abspath LIKE ? ESCAPE '#'
      AND NOT dirindex.deleted
ORDER BY dirindex.abspath ASC""" % Dirent._qualified_columns()
    def subtree(self, abspath, version):
        pattern = (abspath.rstrip("/")
                   .replace("#", "##")
                   .replace("%", "#%")
                   .replace("_", "#_")) + "/%"
        self.execute(self.__subtree, [version, pattern])
        for row in self.cursor:
            yield Dirent._make(row)

    __predecessor = """\
SELECT %s FROM dirindex
WHERE rowid = ?"""  % Dirent._qualified_columns()
    def predecessor(self, dirent):
        assert isinstance(dirent, Dirent)
        if dirent.origin is None:
            return None
        self.execute(self.__predecessor, [dirent.origin])
        return Dirent._make(self.cursor.fetchone())

    __successors = """\
SELECT %s FROM dirindex
where origin = ?
ORDER BY abspath ASC, version ASC""" % Dirent._qualified_columns()
    def successors(self, dirent):
        assert isinstance(dirent, Dirent)
        self.execute(self.__successors, [dirent.rowid])
        for row in self.cursor:
            yield Dirent._make(row)

class Revision(object):
    def __init__(self, index, version,
                 created=None, author=None, log=None):
        self.index = index
        self.version = version
        self.revent = index.get_revision(version)
        self.__created = created
        self.__author = author
        self.__log = log
        self.__context = None
        index.rollback()

    def __enter__(self):
        if self.revent is not None:
            raise Error("revision is read-only")
        logging.log(logging.DEBUG//2, "BEGIN")
        self.revent = self.index.new_revision(
            self.version, self.__created, self.__author, self.__log)
        self.__context = {}
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            if exc_type is None and len(self.__context):
                for dirent in sorted(self.__context.itervalues()):
                    self.index.insert(dirent)
                self.index.commit()
        except:
            self.index.rollback()
            raise
        finally:
            self.__context = None

    def __record(self, dirent, action):
        self.__context[dirent.abspath] = dirent
        if dirent.subtree:
            action = "(%s)" % action
        else:
            action = " %s " % action
        logging.debug(" %-9s %s", action, dirent)

    def __check_writable(self, action):
        if self.__context is None:
            raise Error(action + " requires a transaction")

    def __check_not_root(self, abspath, action):
        if abspath.rstrip("/") == "":
            raise Error(action + " not allowed on /")

    def __find_target(self, abspath, action):
        target = self.__context.get(abspath)
        if target is not None and not target.subtree:
            raise Error(action + " overrides explicit " + abspath)
        if target is None:
            target = self.index.lookup(abspath, self.version - 1)
        if target is None:
            raise Error(action + " target does not exist: " + abspath)
        return target

    def lookup(self, abspath):
        try:
            return self.index.lookup(abspath, self.version)
        finally:
            if self.__context is None:
                self.index.rollback()

    def __add(self, action, abspath, kind, frompath, fromver):
        origin = None
        if frompath is not None:
            origin = self.index.lookup(frompath, fromver)
            if origin is None:
                raise Error(action + " source does not exist: " + frompath)
            if origin.kind != kind:
                raise Error(action + " changes the source object kind")
            origin = origin.rowid
        dirent = Dirent(rowid = None,
                        abspath = abspath,
                        version = self.version,
                        deleted = 0,
                        kind = kind,
                        origin = origin,
                        copied = int(origin is not None),
                        subtree = 0)
        self.__record(dirent, action)
        if frompath is not None:
            offset = len(frompath.rstrip("/"))
            prefix = abspath.rstrip("/")
            for source in self.index.subtree(frompath, fromver):
                dirent = Dirent(rowid = source.rowid,
                                abspath = prefix + source.abspath[offset:],
                                version = self.version,
                                deleted = 0,
                                kind = source.kind,
                                origin = source.rowid,
                                copied = 1,
                                subtree = 1)
                self.__record(dirent, action)

    def add(self, abspath, kind, frompath=None, fromver=None):
        action = "add"
        self.__check_writable(action)
        self.__check_not_root(abspath, action)
        return self.__add(action, abspath, kind, frompath, fromver)

    def replace(self, abspath, kind, frompath=None, fromver=None):
        action = "replace"
        self.__check_writable(action)
        self.__check_not_root(abspath, action)
        self.__find_target(abspath, action)
        return self.__add(action, abspath, kind, frompath, fromver)

    def modify(self, abspath):
        action = "modify"
        self.__check_writable(action)
        target = self.__find_target(abspath, action)
        dirent = Dirent(rowid = None,
                        abspath = abspath,
                        version = self.version,
                        deleted = 0,
                        kind = target.kind,
                        origin = target.rowid,
                        copied = 0,
                        subtree = 0)
        self.__record(dirent, action)

    def delete(self, abspath):
        action = "replace"
        self.__check_writable(action)
        self.__check_not_root(abspath, action)
        target = self.__find_target(abspath, action)
        dirent = Dirent(rowid = None,
                        abspath = abspath,
                        version = self.version,
                        deleted = 1,
                        kind = target.kind,
                        origin = target.rowid,
                        copied = 0,
                        subtree = 0)
        self.__record(dirent, action)
        for source in self.index.subtree(abspath, self.version - 1):
            dirent = Dirent(rowid = source.rowid,
                            abspath = source.abspath,
                            version = self.version,
                            deleted = 1,
                            kind = source.kind,
                            origin = source.rowid,
                            copied = 0,
                            subtree = 1)
            self.__record(dirent, action)

def simpletest(database):
    ix = Index(database)
    ix.initialize()
    with Revision(ix, 1) as rev:
        rev.add(u'/A', 0)
        rev.add(u'/A/B', 0)
        rev.add(u'/A/B/c', 1)
    with Revision(ix, 2) as rev:
        rev.add(u'/A/B/d', 1)
    with Revision(ix, 3) as rev:
        rev.add(u'/X', 0, u'/A', 1)
        rev.add(u'/X/B/d', 1, u'/A/B/d', 2)
    with Revision(ix, 4) as rev:
        rev.delete(u'/X/B/d')
        rev.add(u'/X/B/x', 1, u'/X/B/d', 3)
    with Revision(ix, 5) as rev:
        rev.delete(u'/A')

    for r in (0, 1, 2, 3, 4, 5):
        print "Revision: %d" % r
        for dirent in list(ix.subtree('/', r)):
            origin = ix.predecessor(dirent)
            if origin is None:
                print "   " + str(dirent)
            else:
                print "   %-17s  <- %s" % (dirent, origin)

    dirent = ix.lookup('/A/B/c', 4)
    print "/A/B/c@4 -> %s@%d" % (dirent.abspath, dirent.version)
    for succ in ix.successors(dirent):
        print "%11s %s %s@%d" % (
            "", succ.deleted and "x_x" or "-->",
            succ.abspath, succ.version)

    ix.close()

def loggedsimpletest(database):
    import sys
    logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
    simpletest(database)
<|endoftext|>


<commit_before># Copyright 2014 IBM Corp.
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import math
from nova.compute import arch
from nova.compute import hv_type
from nova.compute import vm_mode
from nova.openstack.common import log as logging

from oslo.serialization import jsonutils

LOG = logging.getLogger(__name__)

# Power VM hypervisor info
IBM_POWERVM_HYPERVISOR_VERSION = '7.1'

# The types of LPARS that are supported.
POWERVM_SUPPORTED_INSTANCES = jsonutils.dumps([(arch.PPC64,
                                                hv_type.PHYP,
                                                vm_mode.HVM),
                                               (arch.PPC64LE,
                                                hv_type.PHYP,
                                                vm_mode.HVM)])

# cpu_info that will be returned by build_host_stats_from_entry()
HOST_STATS_CPU_INFO = jsonutils.dumps({'vendor': 'ibm', 'arch': 'ppc64'})

def build_host_resource_from_ms(ms_wrapper):
    """Build the host resource dict from an MS adapter wrapper

    This method builds the host resource dictionary from the
    ManagedSystem Entry wrapper

    :param ms_wrapper: ManagedSystem Entry Wrapper.
    """
    data = {}

    # Calculate the vcpus
    proc_units = ms_wrapper.proc_units_configurable
    proc_units_avail = ms_wrapper.proc_units_avail
    pu_used = float(proc_units) - float(proc_units_avail)
    data['vcpus'] = int(math.ceil(float(proc_units)))
    data['vcpus_used'] = int(math.ceil(pu_used))

    data['memory_mb'] = ms_wrapper.memory_configurable
    data['memory_mb_used'] = (ms_wrapper.memory_configurable -
                              ms_wrapper.memory_free)

    data["hypervisor_type"] = hv_type.PHYP
    data["hypervisor_version"] = IBM_POWERVM_HYPERVISOR_VERSION
    data["hypervisor_hostname"] = ms_wrapper.mtms.mtms_str()
    data["cpu_info"] = HOST_STATS_CPU_INFO
    data["numa_topology"] = None
    data["supported_instances"] = POWERVM_SUPPORTED_INSTANCES

    stats = {'proc_units': '%.2f' % float(proc_units),
             'proc_units_used': '%.2f' % pu_used
             }
    data["stats"] = stats

    return data
<commit_msg>Update mtms_str to invoke as property<commit_after># Copyright 2014 IBM Corp.
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import math
from nova.compute import arch
from nova.compute import hv_type
from nova.compute import vm_mode
from nova.openstack.common import log as logging

from oslo.serialization import jsonutils

LOG = logging.getLogger(__name__)

# Power VM hypervisor info
IBM_POWERVM_HYPERVISOR_VERSION = '7.1'

# The types of LPARS that are supported.
POWERVM_SUPPORTED_INSTANCES = jsonutils.dumps([(arch.PPC64,
                                                hv_type.PHYP,
                                                vm_mode.HVM),
                                               (arch.PPC64LE,
                                                hv_type.PHYP,
                                                vm_mode.HVM)])

# cpu_info that will be returned by build_host_stats_from_entry()
HOST_STATS_CPU_INFO = jsonutils.dumps({'vendor': 'ibm', 'arch': 'ppc64'})

def build_host_resource_from_ms(ms_wrapper):
    """Build the host resource dict from an MS adapter wrapper

    This method builds the host resource dictionary from the
    ManagedSystem Entry wrapper

    :param ms_wrapper: ManagedSystem Entry Wrapper.
    """
    data = {}

    # Calculate the vcpus
    proc_units = ms_wrapper.proc_units_configurable
    proc_units_avail = ms_wrapper.proc_units_avail
    pu_used = float(proc_units) - float(proc_units_avail)
    data['vcpus'] = int(math.ceil(float(proc_units)))
    data['vcpus_used'] = int(math.ceil(pu_used))

    data['memory_mb'] = ms_wrapper.memory_configurable
    data['memory_mb_used'] = (ms_wrapper.memory_configurable -
                              ms_wrapper.memory_free)

    data["hypervisor_type"] = hv_type.PHYP
    data["hypervisor_version"] = IBM_POWERVM_HYPERVISOR_VERSION
    data["hypervisor_hostname"] = ms_wrapper.mtms.mtms_str
    data["cpu_info"] = HOST_STATS_CPU_INFO
    data["numa_topology"] = None
    data["supported_instances"] = POWERVM_SUPPORTED_INSTANCES

    stats = {'proc_units': '%.2f' % float(proc_units),
             'proc_units_used': '%.2f' % pu_used
             }
    data["stats"] = stats

    return data
<|endoftext|>


<commit_before>"""
Relays sit below an announcer, or another relay, and simply repeat what
they receive over PUB/SUB.
"""
# Logging has to be configured first before we do anything.
import logging

logger = logging.getLogger(__name__)
import zlib
from collections import deque

import gevent
import zmq.green as zmq
from emdr.conf import default_settings as settings
from emdr.daemons.relay.dedupers import is_message_duped

def run():
    """
    Fires up the relay process.
    """
    # These form the connection to the Gateway daemon(s) upstream.
    context = zmq.Context()

    receiver = context.socket(zmq.SUB)
    receiver.setsockopt(zmq.SUBSCRIBE, '')
    for binding in settings.RELAY_RECEIVER_BINDINGS:
        # Relays bind upstream to an Announcer, or another Relay.
        receiver.connect(binding)

    sender = context.socket(zmq.PUB)
    for binding in settings.RELAY_SENDER_BINDINGS:
        # End users, or other relays, may attach here.
        sender.bind(binding)

    def relay_worker(message):
        """
        This is the worker function that re-sends the incoming messages out
        to any subscribers.

        :param str message: A JSON string to re-broadcast.
        """
        if is_message_duped(message):
            # We've already seen this message recently. Discard it.
            return

        if settings.RELAY_DECOMPRESS_MESSAGES:
            message = zlib.decompress(message)

        sender.send(message)
        logger.debug('Message relayed.')

    logger.info("Relay is now listening for order data.")

    while True:
        # For each incoming message, spawn a greenlet using the relay_worker
        # function.
        gevent.spawn(relay_worker, receiver.recv())<commit_msg>Yanking unused import.<commit_after>"""
Relays sit below an announcer, or another relay, and simply repeat what
they receive over PUB/SUB.
"""
# Logging has to be configured first before we do anything.
import logging

logger = logging.getLogger(__name__)
import zlib

import gevent
import zmq.green as zmq
from emdr.conf import default_settings as settings
from emdr.daemons.relay.dedupers import is_message_duped

def run():
    """
    Fires up the relay process.
    """
    # These form the connection to the Gateway daemon(s) upstream.
    context = zmq.Context()

    receiver = context.socket(zmq.SUB)
    receiver.setsockopt(zmq.SUBSCRIBE, '')
    for binding in settings.RELAY_RECEIVER_BINDINGS:
        # Relays bind upstream to an Announcer, or another Relay.
        receiver.connect(binding)

    sender = context.socket(zmq.PUB)
    for binding in settings.RELAY_SENDER_BINDINGS:
        # End users, or other relays, may attach here.
        sender.bind(binding)

    def relay_worker(message):
        """
        This is the worker function that re-sends the incoming messages out
        to any subscribers.

        :param str message: A JSON string to re-broadcast.
        """
        if is_message_duped(message):
            # We've already seen this message recently. Discard it.
            return

        if settings.RELAY_DECOMPRESS_MESSAGES:
            message = zlib.decompress(message)

        sender.send(message)
        logger.debug('Message relayed.')

    logger.info("Relay is now listening for order data.")

    while True:
        # For each incoming message, spawn a greenlet using the relay_worker
        # function.
        gevent.spawn(relay_worker, receiver.recv())<|endoftext|>


<commit_before>import os
import re
import sys
import math
import time
import glob
import shlex
import shutil
import subprocess

from lxml import etree
from datetime import datetime, timedelta

from encarne.helper.config import read_config
from encarne.client.communication import (
    execute_add,
    get_status,
)

def execute_run(args):
    config = read_config()

    args = {key: value for key, value in args.items() if value}
    for key, value in args.items():
        if key == 'directory':
            directory = value
        # Encoding
        if key == 'crf':
            config['encoding']['crf'] = str(value)
        elif key == 'preset':
            config['encoding']['preset'] = value
        elif key == 'audio':
            config['encoding']['audio'] = value
        elif key == 'audio':
            config['encoding']['kbitrate-audio'] = value

    if not directory or not os.path.isdir(directory):
        print('A valid directory needs to be specified')
        print(directory)
        sys.exit(1)
    else:
        # Get absolute path of directory
        directory = os.path.abspath(directory)

    files = find_files(directory)

    processed_files = 0
    for path in files:
        # Get absolute path
        path = os.path.abspath(path)
        # Create media info and get `Writing library` value.
        mediainfo = get_media_encoding(path)
        if 'x265' in mediainfo:
            continue
        processed_files += 1

        # Get directory the movie is in and the name for the temporary
        # new encoded video file.
        directory_path = os.path.dirname(path)
        dest_path = os.path.join(path, '.x265')

        # Compile ffmpeg command
        ffmpeg_command = create_ffmpeg_command(config, path, dest_path)

        # Check if the current command already in the queue.
        index, status = get_current_index(ffmpeg_command)

        # Send the command to pueue for scheduling, if it isn't in the queue yet
        if index is None and status is None:
            # In case a previous run failed and pueue has been resetted,
            # we need to check, if the encoded file is still there.
            if os.path.exists(dest_path):
                os.remove(dest_path)
            args = {
                'command': ffmpeg_command,
                'path': directory_path
            }
            print("Add task pueue:\n {}".format(ffmpeg_command))
            execute_add(args)
        else:
            print("Task already exists in pueue: \n ".format(ffmpeg_command))

        waiting = True
        while waiting:
            # Get index of current command and the current status
            index, status = get_current_index(ffmpeg_command)
            # If the command has been removed or errored,
            # remove the already created destination file
            if (index is None and status is None) or status == 'errored':
                if os.path.exists(dest_path):
                    os.remove(dest_path)
                waiting = False
            # If the command has finished, break the loop for further processing
            elif status == 'done':
                waiting = False

            time.sleep(60)

        if os.path.exists(dest_path):
            print("Pueue task completed")
            copy = True
            # If the destination movie is shorter than a maximum of 20 seconds
            # as the original or has no duration property in mediafile, we drop this.
            origin_duration = get_media_duration(path)
            dest_duration = get_media_duration(dest_path)
            if dest_duration is not None:
                diff = origin_duration - dest_duration
                THRESHOLD = 20
                if math.fabs(diff.total_seconds()) < THRESHOLD:
                    print('Encoded movie is more than {} seconds shorter than original'.format(THRESHOLD))
                    copy = False

            # Check if the filesize of the x.265 encoded object is bigger
            # than the original.
            if copy:
                origin_filesize = os.path.getsize(path)
                dest_filesize = os.path.getsize(dest_path)
                if origin_filesize < dest_filesize:
                    print('Encoded movie is bigger than the original movie')
                    copy = False

            # Only copy if checks above passed
            if copy:
                shutil.move(dest_path, path)
                print("New encoded file is now in place")
            else:
                print("Didn't copy new file, see message above")
        else:
            print("Pueue task failed in some kind of way.")

    if processed_files == 0:
        print('No files for encoding found')

def find_files(path):
    """Get all known video files by recursive extension search."""
    extensions = ['mkv', 'mp4', 'avi']
    files = []
    for extension in extensions:
        found = glob.glob(
            os.path.join(path, '**/*.{}'.format(extension)),
            recursive=True)
        files = files + found
    return files

def create_ffmpeg_command(config, path, dest_path):
    """Compile an ffmpeg command by known parameters."""
    if config['encoding']['kbitrate-audio'] != 'None':
        audio_bitrate = '-b:a {}'.format('audio_bitrate')
    else:
        audio_bitrate = ''
    ffmpeg_command = 'ffmpeg -i {path} -c:v libx265 -preset {preset} ' \
        '-crf {crf} -c:a {audio} {bitrate} {dest}'.format(
            path=shlex.quote(path),
            dest=shlex.quote(dest_path),
            preset=config['encoding']['preset'],
            crf=config['encoding']['crf'],
            audio=config['encoding']['audio'],
            bitrate=audio_bitrate,
        )
    return ffmpeg_command

def get_media_encoding(path):
    """Execute external mediainfo command and find the video encoding library."""
    process = subprocess.run(
        ['mediainfo', '--Output=XML', path],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    root = etree.XML(process.stdout)
    writing_library = root.findall('.//track[@type="Video"]/Writing_library')[0].text

    return writing_library

def get_media_duration(path):
    """Execute external mediainfo command and find the video encoding library."""
    process = subprocess.run(
        ['mediainfo', '--Output=XML', path],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    root = etree.XML(process.stdout)
    try:
        time = root.findall('.//track[@type="General"]/Duration')[0].text
    except IndexError:
        # No duration, we return None
        return None

    match = re.match(r'\d{0,2} h \d{0,2} min \d{0,2} s', time)
    if match:
        date = datetime.strptime(time, '%H h %M min %S s')
    if not match:
        match = re.match(r'\d{0,2} h \d{0,2} min', time)
        if match:
            date = datetime.strptime(time, '%H h %M min')
    if not match:
        match = re.match(r'\d{0,2} min \d{0,2} s', time)
        if match:
            date = datetime.strptime(time, '%M min %S s')
    if not match:
        print("No known time format: {}".format(time))
        return None

    delta = timedelta(
        hours=date.hour,
        minutes=date.minute,
        seconds=date.second,
    )

    return delta

def get_current_index(command):
    """Get the status and key of the given process in pueue."""
    status = get_status()

    if isinstance(status['data'], dict):
        # Get the status of the latest submitted job.
        smallest_key = None
        for key, value in status['data'].items():
            if value['command'] == command:
                if smallest_key is None or smallest_key < key:
                    smallest_key = key
        if smallest_key is not None:
            return smallest_key, status['data'][key]['status']
    return None, None
<commit_msg>Fix filepath<commit_after>import os
import re
import sys
import math
import time
import glob
import shlex
import shutil
import subprocess

from lxml import etree
from datetime import datetime, timedelta

from encarne.helper.config import read_config
from encarne.client.communication import (
    execute_add,
    get_status,
)

def execute_run(args):
    config = read_config()

    args = {key: value for key, value in args.items() if value}
    for key, value in args.items():
        if key == 'directory':
            directory = value
        # Encoding
        if key == 'crf':
            config['encoding']['crf'] = str(value)
        elif key == 'preset':
            config['encoding']['preset'] = value
        elif key == 'audio':
            config['encoding']['audio'] = value
        elif key == 'audio':
            config['encoding']['kbitrate-audio'] = value

    if not directory or not os.path.isdir(directory):
        print('A valid directory needs to be specified')
        print(directory)
        sys.exit(1)
    else:
        # Get absolute path of directory
        directory = os.path.abspath(directory)

    files = find_files(directory)

    processed_files = 0
    for path in files:
        # Get absolute path
        path = os.path.abspath(path)
        # Create media info and get `Writing library` value.
        mediainfo = get_media_encoding(path)
        if 'x265' in mediainfo:
            continue
        processed_files += 1

        # Get directory the movie is in and the name for the temporary
        # new encoded video file.
        directory_path = os.path.dirname(path)
        dest_path = os.path.splitext(path)[0] + '-x265.mkv'

        # Compile ffmpeg command
        ffmpeg_command = create_ffmpeg_command(config, path, dest_path)

        # Check if the current command already in the queue.
        index, status = get_current_index(ffmpeg_command)

        # Send the command to pueue for scheduling, if it isn't in the queue yet
        if index is None and status is None:
            # In case a previous run failed and pueue has been resetted,
            # we need to check, if the encoded file is still there.
            if os.path.exists(dest_path):
                os.remove(dest_path)
            args = {
                'command': ffmpeg_command,
                'path': directory_path
            }
            print("Add task pueue:\n {}".format(ffmpeg_command))
            execute_add(args)
        else:
            print("Task already exists in pueue: \n ".format(ffmpeg_command))

        waiting = True
        while waiting:
            # Get index of current command and the current status
            index, status = get_current_index(ffmpeg_command)
            # If the command has been removed or errored,
            # remove the already created destination file
            if (index is None and status is None) or status == 'errored':
                if os.path.exists(dest_path):
                    os.remove(dest_path)
                waiting = False
            # If the command has finished, break the loop for further processing
            elif status == 'done':
                waiting = False

            time.sleep(60)

        if os.path.exists(dest_path):
            print("Pueue task completed")
            copy = True
            # If the destination movie is shorter than a maximum of 20 seconds
            # as the original or has no duration property in mediafmkvile, we drop this.
            origin_duration = get_media_duration(path)
            dest_duration = get_media_duration(dest_path)
            if dest_duration is not None:
                diff = origin_duration - dest_duration
                THRESHOLD = 20
                if math.fabs(diff.total_seconds()) < THRESHOLD:
                    print('Encoded movie is more than {} seconds shorter than original'.format(THRESHOLD))
                    copy = False

            # Check if the filesize of the x.265 encoded object is bigger
            # than the original.
            if copy:
                origin_filesize = os.path.getsize(path)
                dest_filesize = os.path.getsize(dest_path)
                if origin_filesize < dest_filesize:
                    print('Encoded movie is bigger than the original movie')
                    copy = False

            # Only copy if checks above passed
            if copy:
                shutil.move(dest_path, path)
                print("New encoded file is now in place")
            else:
                print("Didn't copy new file, see message above")
        else:
            print("Pueue task failed in some kind of way.")

    if processed_files == 0:
        print('No files for encoding found')

def find_files(path):
    """Get all known video files by recursive extension search."""
    extensions = ['mkv', 'mp4', 'avi']
    files = []
    for extension in extensions:
        found = glob.glob(
            os.path.join(path, '**/*.{}'.format(extension)),
            recursive=True)
        files = files + found
    return files

def create_ffmpeg_command(config, path, dest_path):
    """Compile an ffmpeg command by known parameters."""
    if config['encoding']['kbitrate-audio'] != 'None':
        audio_bitrate = '-b:a {}'.format('audio_bitrate')
    else:
        audio_bitrate = ''
    ffmpeg_command = 'ffmpeg -i {path} -c:v libx265 -preset {preset} ' \
        '-crf {crf} -c:a {audio} {bitrate} {dest}'.format(
            path=shlex.quote(path),
            dest=shlex.quote(dest_path),
            preset=config['encoding']['preset'],
            crf=config['encoding']['crf'],
            audio=config['encoding']['audio'],
            bitrate=audio_bitrate,
        )
    return ffmpeg_command

def get_media_encoding(path):
    """Execute external mediainfo command and find the video encoding library."""
    process = subprocess.run(
        ['mediainfo', '--Output=XML', path],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    root = etree.XML(process.stdout)
    writing_library = root.findall('.//track[@type="Video"]/Writing_library')[0].text

    return writing_library

def get_media_duration(path):
    """Execute external mediainfo command and find the video encoding library."""
    process = subprocess.run(
        ['mediainfo', '--Output=XML', path],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    root = etree.XML(process.stdout)
    try:
        time = root.findall('.//track[@type="General"]/Duration')[0].text
    except IndexError:
        # No duration, we return None
        return None

    match = re.match(r'\d{0,2} h \d{0,2} min \d{0,2} s', time)
    if match:
        date = datetime.strptime(time, '%H h %M min %S s')
    if not match:
        match = re.match(r'\d{0,2} h \d{0,2} min', time)
        if match:
            date = datetime.strptime(time, '%H h %M min')
    if not match:
        match = re.match(r'\d{0,2} min \d{0,2} s', time)
        if match:
            date = datetime.strptime(time, '%M min %S s')
    if not match:
        print("No known time format: {}".format(time))
        return None

    delta = timedelta(
        hours=date.hour,
        minutes=date.minute,
        seconds=date.second,
    )

    return delta

def get_current_index(command):
    """Get the status and key of the given process in pueue."""
    status = get_status()

    if isinstance(status['data'], dict):
        # Get the status of the latest submitted job.
        smallest_key = None
        for key, value in status['data'].items():
            if value['command'] == command:
                if smallest_key is None or smallest_key < key:
                    smallest_key = key
        if smallest_key is not None:
            return smallest_key, status['data'][key]['status']
    return None, None
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from __future__ import print_function, unicode_literals, division
import logging

from enocean.protocol import crc8
from enocean.protocol.eep import EEP
from enocean.protocol.constants import PACKET, RORG, PARSE_RESULT

logger = logging.getLogger('enocean.protocol.packet')
eep = EEP()

class Packet(object):
    '''
    Base class for Packet.
    Mainly used for for packet generation and
    Packet.parse_msg(buf) for parsing message.
    parse_msg() returns subclass, if one is defined for the data type.
    '''
    def __init__(self, type, data=[], optional=[]):
        self.type = type
        self.rorg = RORG.UNDEFINED
        self.rorg_func = None
        self.rorg_type = None
        self.data = data
        self.optional = optional
        self.bit_data = []
        self.bit_optional = []
        self.parsed = {}

        self.parse()

    def __str__(self):
        return '0x%02X %s %s %s' % (self.type, [hex(o) for o in self.data], [hex(o) for o in self.optional], self.parsed)

    def __unicode__(self):
        return self.__str__()

    def __eq__(self, other):
        return self.type == other.type and self.rorg == other.rorg and self.data == other.data and self.optional == other.optional

    def _get_bit(self, byte, bit):
        ''' Get bit value from byte '''
        return (byte >> bit) & 0x01

    def _combine_hex(self, data):
        ''' Combine list of integer values to one big integer '''
        output = 0x00
        for i, d in enumerate(reversed(data)):
            output |= (d << i * 8)
        return output

    def _to_bitarray(self, data, width=8):
        ''' Convert data (list of integers, bytearray or integer) to bitarray '''
        if isinstance(data, list) or isinstance(data, bytearray):
            data = self._combine_hex(data)
        return [True if digit == '1' else False for digit in bin(data)[2:].zfill(width)]

    @staticmethod
    def parse_msg(buf):
        '''
        Parses message from buffer.
        returns:
            - PARSE_RESULT
            - remaining buffer
            - Packet -object (if message was valid, else None)
        '''
        # If the buffer doesn't contain 0x55 (start char)
        # the message isn't needed -> ignore
        if 0x55 not in buf:
            return PARSE_RESULT.INCOMPLETE, [], None

        # Valid buffer starts from 0x55
        # Convert to list, as index -method isn't defined for bytearray
        buf = buf[list(buf).index(0x55):]
        try:
            data_len = (buf[1] << 8) | buf[2]
            opt_len = buf[3]
        except IndexError:
            # If the fields don't exist, message is incomplete
            return PARSE_RESULT.INCOMPLETE, buf, None

        # Header: 6 bytes, data, optional data and data checksum
        msg_len = 6 + data_len + opt_len + 1
        if len(buf) < msg_len:
            # If buffer isn't long enough, the message is incomplete
            return PARSE_RESULT.INCOMPLETE, buf, None

        msg = buf[0:msg_len]
        buf = buf[msg_len:]

        packet_type = msg[4]
        data = msg[6:6 + data_len]
        opt_data = msg[6 + data_len:6 + data_len + opt_len]

        # Check CRCs for header and data
        if msg[5] != crc8.calc(msg[1:5]):
            # Fail if doesn't match message
            logger.error('Header CRC error!')
            # Return CRC_MISMATCH
            return PARSE_RESULT.CRC_MISMATCH, buf, None
        if msg[6 + data_len + opt_len] != crc8.calc(msg[6:6 + data_len + opt_len]):
            # Fail if doesn't match message
            logger.error('Data CRC error!')
            # Return CRC_MISMATCH
            return PARSE_RESULT.CRC_MISMATCH, buf, None

        # If we got this far, everything went ok (?)
        if packet_type == PACKET.RADIO:
            p = RadioPacket(packet_type, data, opt_data)
        elif packet_type == PACKET.RESPONSE:
            p = ResponsePacket(packet_type, data, opt_data)
        else:
            p = Packet(packet_type, data, opt_data)

        return PARSE_RESULT.OK, buf, p

    def parse(self):
        ''' Parse data from Packet '''
        return self.parsed

    def parse_eep(self, func, type):
        ''' Parse EEP based on FUNC and TYPE '''
        provides, values = eep.get_values(self.rorg, func, type, self.bit_data)
        self.parsed.update(values)
        return list(provides)

    def build(self):
        ''' Build Packet for sending to EnOcean controller '''
        data_length = len(self.data)
        ords = [0x55, (data_length >> 8) & 0xFF, data_length & 0xFF, len(self.optional), int(self.type)]
        ords.append(crc8.calc(ords[1:5]))
        ords.extend(self.data)
        ords.extend(self.optional)
        ords.append(crc8.calc(ords[6:]))
        return ords

class RadioPacket(Packet):
    destination = 0
    destination_hex = ''
    dBm = 0
    status = 0
    sender = 0
    sender_hex = ''
    learn = True
    contains_eep = False

    def __str__(self):
        packet_str = super(RadioPacket, self).__str__()
        return '%s->%s (%d dBm): %s' % (self.sender_hex, self.destination_hex, self.dBm, packet_str)

    def parse(self):
        self.destination = self._combine_hex(self.optional[1:5])
        self.destination_hex = ':'.join([('%02X' % o) for o in self.optional[1:5]])
        self.dBm = -self.optional[5]
        self.status = self.data[-1]
        self.sender = self._combine_hex(self.data[-5:-1])
        self.sender_hex = ':'.join([('%02X' % o) for o in self.data[-5:-1]])
        # Default to learn == True, as some devices don't have a learn button
        self.learn = True

        self.rorg = self.data[0]
        if self.rorg == RORG.RPS:
            self.bit_data = self._to_bitarray(self.data[1], 8)
        if self.rorg == RORG.BS1:
            self.bit_data = self._to_bitarray(self.data[1], 8)
            self.learn = not self.bit_data[-4]
        if self.rorg == RORG.BS4:
            self.bit_data = self._to_bitarray(self.data[1:5], 32)
            self.learn = not self.bit_data[-4]
            if self.learn:
                self.contains_eep = self.bit_data[-8]
                if self.contains_eep:
                    # Get rorg_func and rorg_type from an unidirectionnal learn packet
                    self.rorg_func = self.data[1] >> 2
                    self.rorg_type = ((self.data[1] & 0x03) << 5) | (self.data[2] >> 3)

        return super(RadioPacket, self).parse()

class ResponsePacket(Packet):
    response = 0
    response_data = []

    def parse(self):
        self.response = self.data[0]
        self.response_data = self.data[1:]
        return super(ResponsePacket, self).parse()

class EventPacket(Packet):
    event = 0
    event_data = []

    def parse(self):
        self.event = self.data[0]
        self.event_data = self.data[1:]
        return super(EventPacket, self).parse()
<commit_msg>Try to parse EEP automatically, if provided in the message.<commit_after># -*- encoding: utf-8 -*-
from __future__ import print_function, unicode_literals, division
import logging

from enocean.protocol import crc8
from enocean.protocol.eep import EEP
from enocean.protocol.constants import PACKET, RORG, PARSE_RESULT

logger = logging.getLogger('enocean.protocol.packet')
eep = EEP()

class Packet(object):
    '''
    Base class for Packet.
    Mainly used for for packet generation and
    Packet.parse_msg(buf) for parsing message.
    parse_msg() returns subclass, if one is defined for the data type.
    '''
    def __init__(self, type, data=[], optional=[]):
        self.type = type
        self.rorg = RORG.UNDEFINED
        self.rorg_func = None
        self.rorg_type = None
        self.data = data
        self.optional = optional
        self.bit_data = []
        self.bit_optional = []
        self.parsed = {}

        self.parse()

    def __str__(self):
        return '0x%02X %s %s %s' % (self.type, [hex(o) for o in self.data], [hex(o) for o in self.optional], self.parsed)

    def __unicode__(self):
        return self.__str__()

    def __eq__(self, other):
        return self.type == other.type and self.rorg == other.rorg and self.data == other.data and self.optional == other.optional

    def _get_bit(self, byte, bit):
        ''' Get bit value from byte '''
        return (byte >> bit) & 0x01

    def _combine_hex(self, data):
        ''' Combine list of integer values to one big integer '''
        output = 0x00
        for i, d in enumerate(reversed(data)):
            output |= (d << i * 8)
        return output

    def _to_bitarray(self, data, width=8):
        ''' Convert data (list of integers, bytearray or integer) to bitarray '''
        if isinstance(data, list) or isinstance(data, bytearray):
            data = self._combine_hex(data)
        return [True if digit == '1' else False for digit in bin(data)[2:].zfill(width)]

    @staticmethod
    def parse_msg(buf):
        '''
        Parses message from buffer.
        returns:
            - PARSE_RESULT
            - remaining buffer
            - Packet -object (if message was valid, else None)
        '''
        # If the buffer doesn't contain 0x55 (start char)
        # the message isn't needed -> ignore
        if 0x55 not in buf:
            return PARSE_RESULT.INCOMPLETE, [], None

        # Valid buffer starts from 0x55
        # Convert to list, as index -method isn't defined for bytearray
        buf = buf[list(buf).index(0x55):]
        try:
            data_len = (buf[1] << 8) | buf[2]
            opt_len = buf[3]
        except IndexError:
            # If the fields don't exist, message is incomplete
            return PARSE_RESULT.INCOMPLETE, buf, None

        # Header: 6 bytes, data, optional data and data checksum
        msg_len = 6 + data_len + opt_len + 1
        if len(buf) < msg_len:
            # If buffer isn't long enough, the message is incomplete
            return PARSE_RESULT.INCOMPLETE, buf, None

        msg = buf[0:msg_len]
        buf = buf[msg_len:]

        packet_type = msg[4]
        data = msg[6:6 + data_len]
        opt_data = msg[6 + data_len:6 + data_len + opt_len]

        # Check CRCs for header and data
        if msg[5] != crc8.calc(msg[1:5]):
            # Fail if doesn't match message
            logger.error('Header CRC error!')
            # Return CRC_MISMATCH
            return PARSE_RESULT.CRC_MISMATCH, buf, None
        if msg[6 + data_len + opt_len] != crc8.calc(msg[6:6 + data_len + opt_len]):
            # Fail if doesn't match message
            logger.error('Data CRC error!')
            # Return CRC_MISMATCH
            return PARSE_RESULT.CRC_MISMATCH, buf, None

        # If we got this far, everything went ok (?)
        if packet_type == PACKET.RADIO:
            p = RadioPacket(packet_type, data, opt_data)
        elif packet_type == PACKET.RESPONSE:
            p = ResponsePacket(packet_type, data, opt_data)
        else:
            p = Packet(packet_type, data, opt_data)

        return PARSE_RESULT.OK, buf, p

    def parse(self):
        ''' Parse data from Packet '''
        return self.parsed

    def parse_eep(self, func, type):
        ''' Parse EEP based on FUNC and TYPE '''
        provides, values = eep.get_values(self.rorg, func, type, self.bit_data)
        self.parsed.update(values)
        return list(provides)

    def build(self):
        ''' Build Packet for sending to EnOcean controller '''
        data_length = len(self.data)
        ords = [0x55, (data_length >> 8) & 0xFF, data_length & 0xFF, len(self.optional), int(self.type)]
        ords.append(crc8.calc(ords[1:5]))
        ords.extend(self.data)
        ords.extend(self.optional)
        ords.append(crc8.calc(ords[6:]))
        return ords

class RadioPacket(Packet):
    destination = 0
    destination_hex = ''
    dBm = 0
    status = 0
    sender = 0
    sender_hex = ''
    learn = True
    contains_eep = False

    def __str__(self):
        packet_str = super(RadioPacket, self).__str__()
        return '%s->%s (%d dBm): %s' % (self.sender_hex, self.destination_hex, self.dBm, packet_str)

    def parse(self):
        self.destination = self._combine_hex(self.optional[1:5])
        self.destination_hex = ':'.join([('%02X' % o) for o in self.optional[1:5]])
        self.dBm = -self.optional[5]
        self.status = self.data[-1]
        self.sender = self._combine_hex(self.data[-5:-1])
        self.sender_hex = ':'.join([('%02X' % o) for o in self.data[-5:-1]])
        # Default to learn == True, as some devices don't have a learn button
        self.learn = True

        self.rorg = self.data[0]
        if self.rorg == RORG.RPS:
            self.bit_data = self._to_bitarray(self.data[1], 8)
        if self.rorg == RORG.BS1:
            self.bit_data = self._to_bitarray(self.data[1], 8)
            self.learn = not self.bit_data[-4]
        if self.rorg == RORG.BS4:
            self.bit_data = self._to_bitarray(self.data[1:5], 32)
            self.learn = not self.bit_data[-4]
            if self.learn:
                self.contains_eep = self.bit_data[-8]
                if self.contains_eep:
                    # Get rorg_func and rorg_type from an unidirectional learn packet
                    self.rorg_func = self.data[1] >> 2
                    self.rorg_type = ((self.data[1] & 0x03) << 5) | (self.data[2] >> 3)
                    # Try to parse by EEP
                    self.parse_eep(self.rorg_func, self.rorg_type)

        return super(RadioPacket, self).parse()

class ResponsePacket(Packet):
    response = 0
    response_data = []

    def parse(self):
        self.response = self.data[0]
        self.response_data = self.data[1:]
        return super(ResponsePacket, self).parse()

class EventPacket(Packet):
    event = 0
    event_data = []

    def parse(self):
        self.event = self.data[0]
        self.event_data = self.data[1:]
        return super(EventPacket, self).parse()
<|endoftext|>


<commit_before>import ugfx
import badge
import sys
import gc
import uos as os
import uerrno as errno
import ujson as json
import ussl
import usocket
import time
import network

sta_if = network.WLAN(network.STA_IF); sta_if.active(True)
badge.wifi_init()

ugfx.clear(ugfx.BLACK)
ugfx.string(20,25,"Connecting to:","Roboto_BlackItalic24",ugfx.WHITE)
ugfx.string(140,75, "WiFi","PermanentMarker22",ugfx.WHITE)
ugfx.flush()

while not sta_if.isconnected():
    time.sleep(0.1)
    pass

ugfx.clear(ugfx.WHITE)
ugfx.string(180,25,"STILL","Roboto_BlackItalic24",ugfx.BLACK)
ugfx.string(160,50,"Hacking","PermanentMarker22",ugfx.BLACK)
len = ugfx.get_string_width("Hacking","PermanentMarker22")
ugfx.line(160, 72, 144 + len, 72, ugfx.BLACK)
ugfx.line(170 + len, 52, 170 + len, 70, ugfx.BLACK)
ugfx.string(170,75,"Anyway","Roboto_BlackItalic24",ugfx.BLACK)

ugfx.input_init()

debug = False
warn_ussl = True

def url_open(url):
    global warn_ussl

    if debug:
        print(url)

    proto, _, host, urlpath = url.split('/', 3)
    try:
        ai = usocket.getaddrinfo(host, 443)
    except OSError as e:
        fatal("Unable to resolve %s (no Internet?)" % host, e)
    #print("Address infos:", ai)
    addr = ai[0][4]

    s = usocket.socket(ai[0][0])
    try:
        #print("Connect address:", addr)
        s.connect(addr)

        if proto == "https:":
            s = ussl.wrap_socket(s, server_hostname=host)
            if warn_ussl:
                print("Warning: %s SSL certificate is not validated" % host)
                warn_ussl = False

        # MicroPython rawsocket module supports file interface directly
        s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
        l = s.readline()
        protover, status, msg = l.split(None, 2)
        if status != b"200":
            if status == b"404" or status == b"301":
                raise NotFoundError("Package not found")
            raise ValueError(status)
        while 1:
            l = s.readline()
            if not l:
                raise ValueError("Unexpected EOF in HTTP headers")
            if l == b'\r\n':
                break
    except Exception as e:
        s.close()
        raise e

    return s

options = ugfx.List(0,0,int(ugfx.width()/2),ugfx.height())

f = url_open("https://badge.sha2017.org/eggs/list/json")
try:
    packages = json.load(f)
finally:
    f.close()

for package in packages:
    options.add_item("%s rev. %s"% (package["name"], package["revision"]))

ugfx.set_lut(ugfx.LUT_FULL)
ugfx.flush()

def show_description(active):
    if active:
         text.text(packages[options.selected_index()]["description"])
    ugfx.flush()

def woezel_it(active):
    if active:
        ugfx.clear(ugfx.BLACK)
        ugfx.string(40,25,"Installing:","Roboto_BlackItalic24",ugfx.WHITE)
        ugfx.string(100,75, packages[options.selected_index()]["name"],"PermanentMarker22",ugfx.WHITE)
        ugfx.flush()
        import woezel
        woezel.install(packages[options.selected_index()]["slug"])
        ugfx.clear(ugfx.BLACK)
        ugfx.string(40,25,"Running:","Roboto_BlackItalic24",ugfx.WHITE)
        ugfx.string(100,75, packages[options.selected_index()]["name"],"PermanentMarker22",ugfx.WHITE)
        ugfx.flush()
        app = __import__(packages[options.selected_index()]["slug"])

ugfx.input_attach(ugfx.JOY_UP, show_description)
ugfx.input_attach(ugfx.JOY_DOWN, show_description)
ugfx.input_attach(ugfx.BTN_A, woezel_it)
ugfx.input_attach(ugfx.BTN_B, woezel_it)

ugfx.input_attach(ugfx.JOY_LEFT, lambda x: ugfx.flush())
ugfx.input_attach(ugfx.JOY_RIGHT, lambda x: ugfx.flush())

text = ugfx.Textbox(int(ugfx.width()/2),0, int(ugfx.width()/2), ugfx.height())

ugfx.flush()
time.sleep(0.2)
ugfx.set_lut(ugfx.LUT_FASTER)
<commit_msg>Updated colour scheme<commit_after>import ugfx
import badge
import sys
import gc
import uos as os
import uerrno as errno
import ujson as json
import ussl
import usocket
import time
import network

sta_if = network.WLAN(network.STA_IF); sta_if.active(True)
badge.wifi_init()

ugfx.clear(ugfx.BLACK)
ugfx.string(20,25,"Connecting to:","Roboto_BlackItalic24",ugfx.WHITE)
ugfx.string(140,75, "WiFi","PermanentMarker22",ugfx.WHITE)
ugfx.flush()

while not sta_if.isconnected():
    time.sleep(0.1)
    pass

ugfx.clear(ugfx.WHITE)
ugfx.string(180,25,"STILL","Roboto_BlackItalic24",ugfx.BLACK)
ugfx.string(160,50,"Hacking","PermanentMarker22",ugfx.BLACK)
str_len = ugfx.get_string_width("Hacking","PermanentMarker22")
ugfx.line(160, 72, 144 + str_len, 72, ugfx.BLACK)
ugfx.line(170 + str_len, 52, 170 + str_len, 70, ugfx.BLACK)
ugfx.string(170,75,"Anyway","Roboto_BlackItalic24",ugfx.BLACK)

ugfx.input_init()

debug = False
warn_ussl = True

def url_open(url):
    global warn_ussl

    if debug:
        print(url)

    proto, _, host, urlpath = url.split('/', 3)
    try:
        ai = usocket.getaddrinfo(host, 443)
    except OSError as e:
        fatal("Unable to resolve %s (no Internet?)" % host, e)
    #print("Address infos:", ai)
    addr = ai[0][4]

    s = usocket.socket(ai[0][0])
    try:
        #print("Connect address:", addr)
        s.connect(addr)

        if proto == "https:":
            s = ussl.wrap_socket(s, server_hostname=host)
            if warn_ussl:
                print("Warning: %s SSL certificate is not validated" % host)
                warn_ussl = False

        # MicroPython rawsocket module supports file interface directly
        s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
        l = s.readline()
        protover, status, msg = l.split(None, 2)
        if status != b"200":
            if status == b"404" or status == b"301":
                raise NotFoundError("Package not found")
            raise ValueError(status)
        while 1:
            l = s.readline()
            if not l:
                raise ValueError("Unexpected EOF in HTTP headers")
            if l == b'\r\n':
                break
    except Exception as e:
        s.close()
        raise e

    return s

options = ugfx.List(0,0,int(ugfx.width()/2),ugfx.height())

f = url_open("https://badge.sha2017.org/eggs/list/json")
try:
    packages = json.load(f)
finally:
    f.close()

for package in packages:
    options.add_item("%s rev. %s"% (package["name"], package["revision"]))

ugfx.set_lut(ugfx.LUT_FULL)
ugfx.flush()

def show_description(active):
    if active:
         text.text(packages[options.selected_index()]["description"])
    ugfx.flush()

def woezel_it(active):
    if active:
        ugfx.clear(ugfx.BLACK)
        ugfx.string(40,25,"Installing:","Roboto_BlackItalic24",ugfx.WHITE)
        ugfx.string(100,75, packages[options.selected_index()]["name"],"PermanentMarker22",ugfx.WHITE)
        ugfx.flush()
        import woezel
        woezel.install(packages[options.selected_index()]["slug"])
        ugfx.clear(ugfx.BLACK)
        ugfx.string(40,25,"Running:","Roboto_BlackItalic24",ugfx.WHITE)
        ugfx.string(100,75, packages[options.selected_index()]["name"],"PermanentMarker22",ugfx.WHITE)
        ugfx.flush()
        app = __import__(packages[options.selected_index()]["slug"])

ugfx.input_attach(ugfx.JOY_UP, show_description)
ugfx.input_attach(ugfx.JOY_DOWN, show_description)
ugfx.input_attach(ugfx.BTN_A, woezel_it)
ugfx.input_attach(ugfx.BTN_B, woezel_it)

ugfx.input_attach(ugfx.JOY_LEFT, lambda x: ugfx.flush())
ugfx.input_attach(ugfx.JOY_RIGHT, lambda x: ugfx.flush())

text = ugfx.Textbox(int(ugfx.width()/2),0, int(ugfx.width()/2), ugfx.height())

ugfx.flush()
time.sleep(0.2)
ugfx.set_lut(ugfx.LUT_FASTER)
<|endoftext|>


<commit_before><commit_msg>Switching from QTextEdit to QLabel for better scrolling behavior with the SMS Window<commit_after><|endoftext|>


<commit_before><commit_msg>Some missinglogging<commit_after><|endoftext|>


<commit_before>from ethel.chroot import schroot, copy, scmd
from storz.wrapper import generate_analysis
from firehose.model import Issue, Message, File, Location

import os
import re

OUTPUT_REGEX = re.compile(r"(?P<package>.*): (?P<tag>[^\s]*) (?P<info>.*)")

def parse_output(lines):
    for line in lines:
        info = OUTPUT_REGEX.match(line).groupdict()

        testid = info['tag']
        severity = "error"
        pth = info['info'].split(" ", 1)
        pth = pth[0] if pth else None

        if pth is None:
            continue

        yield Issue(cwe=None,
                    testid=testid,
                    location=Location(file=File(pth, None),
                                      function=None,
                                      point=None),
                    severity=severity,
                    message=Message(text=line),
                    notes=None,
                    trace=None)

def adequate(chroot, package):
    analysis = generate_analysis("adequate", "unstable", package)

    with schroot(chroot) as session:
        deb = os.path.basename(package)
        if not deb.endswith('.deb'):
            raise ValueError("Stop with the crack smoking")

        where = '/tmp/%s' % (deb)
        copy(session, package, where)

        out, err = scmd(session, [
            'apt-get', 'install', '-y', 'adequate'
        ], user='root')

        out, err = scmd(session, [
            'dpkg', '-i', where
        ], user='root', expected=(0, 1))

        out, err = scmd(session, [
            'apt-get', 'install', '-y', '-f'
        ], user='root')

        out, err = scmd(session, ['adequate', deb.split("_", 1)[0]])
        failed = False
        for issue in parse_output(out.splitlines()):
            failed = True
            analysis.results.append(issue)

        return failed, out, analysis

def main():
    import sys
    output = open(sys.argv[3], 'wb')
    info = adequate(sys.argv[1], sys.argv[2])
    output.write(info.to_xml_bytes())
<commit_msg>fixing the wrapper<commit_after>from ethel.chroot import schroot, copy, scmd
from ethel.wrappers.adequate import parse_adequate

from firehose.model import Issue, Message, File, Location
from storz.wrapper import generate_analysis

import os

def adequate(chroot, package):
    analysis = generate_analysis("adequate", "unstable", package)

    with schroot(chroot) as session:
        deb = os.path.basename(package)
        if not deb.endswith('.deb'):
            raise ValueError("Stop with the crack smoking")

        where = '/tmp/%s' % (deb)
        copy(session, package, where)

        out, err = scmd(session, [
            'apt-get', 'install', '-y', 'adequate'
        ], user='root')

        out, err = scmd(session, [
            'dpkg', '-i', where
        ], user='root', expected=(0, 1))

        out, err = scmd(session, [
            'apt-get', 'install', '-y', '-f'
        ], user='root')

        out, err = scmd(session, ['adequate', deb.split("_", 1)[0]])
        failed = False
        for issue in parse_adequate(out.splitlines()):
            failed = True
            analysis.results.append(issue)

        return failed, out, analysis

def main():
    import sys
    output = open(sys.argv[3], 'wb')
    failed, out, analysis = adequate(sys.argv[1], sys.argv[2])
    output.write(analysis.to_xml_bytes())
<|endoftext|>