<commit_before>import logging
import random
import time
from coapthon.messages.message import Message
from coapthon import defines
from coapthon.messages.request import Request
from coapthon.transaction import Transaction

logger = logging.getLogger(__name__)

class MessageLayer(object):
    def __init__(self, starting_mid):
        self._transactions = {}
        self._transactions_token = {}
        if starting_mid is not None:
            self._current_mid = starting_mid
        else:
            self._current_mid = random.randint(1, 1000)

    def purge(self):
        for k in self._transactions.keys():
            now = time.time()
            transaction = self._transactions[k]
            if transaction.timestamp + defines.EXCHANGE_LIFETIME < now:
                logger.debug("Delete transaction")
                del self._transactions[k]
        for k in self._transactions_token.keys():
            now = time.time()
            transaction = self._transactions_token[k]
            if transaction.timestamp + defines.EXCHANGE_LIFETIME < now:
                logger.debug("Delete transaction")
                del self._transactions_token[k]

    def receive_request(self, request):
        """

        :type request: Request
        :param request: the incoming request
        :rtype : Transaction
        """
        logger.debug("receive_request - " + str(request))
        try:
            host, port = request.source
        except AttributeError:
            return
        key_mid = hash(str(host).lower() + str(port).lower() + str(request.mid).lower())
        key_token = hash(str(host).lower() + str(port).lower() + str(request.token).lower())

        if key_mid in self._transactions.keys():
            # Duplicated
            self._transactions[key_mid].request.duplicated = True
            transaction = self._transactions[key_mid]
        else:
            request.timestamp = time.time()
            transaction = Transaction(request=request, timestamp=request.timestamp)
            with transaction:
                self._transactions[key_mid] = transaction
                self._transactions_token[key_token] = transaction
        return transaction

    def receive_response(self, response):
        """

        :type response: Response
        :param response:
        :rtype : Transaction
        """
        logger.debug("receive_response - " + str(response))
        try:
            host, port = response.source
        except AttributeError:
            return
        key_mid = hash(str(host).lower() + str(port).lower() + str(response.mid).lower())
        key_mid_multicast = hash(str(defines.ALL_COAP_NODES).lower() + str(port).lower() + str(response.mid).lower())
        key_token = hash(str(host).lower() + str(port).lower() + str(response.token).lower())
        key_token_multicast = hash(str(defines.ALL_COAP_NODES).lower() + str(port).lower() + str(response.token).lower())
        if key_mid in self._transactions.keys():
            transaction = self._transactions[key_mid]
        elif key_token in self._transactions_token:
            transaction = self._transactions_token[key_token]
        elif key_mid_multicast in self._transactions.keys():
            transaction = self._transactions[key_mid_multicast]
        elif key_token_multicast in self._transactions_token:
            transaction = self._transactions_token[key_token_multicast]
        else:
            logger.warning("Un-Matched incoming response message " + str(host) + ":" + str(port))
            return None, False
        send_ack = False
        if response.type == defines.Types["CON"]:
            send_ack = True

        transaction.request.acknowledged = True
        transaction.completed = True
        transaction.response = response
        if transaction.retransmit_stop is not None:
            transaction.retransmit_stop.set()
        return transaction, send_ack

    def receive_empty(self, message):
        """

        :type message: Message
        :param message:
        :rtype : Transaction
        """
        logger.debug("receive_empty - " + str(message))
        try:
            host, port = message.source
        except AttributeError:
            return
        key_mid = hash(str(host) + str(port) + str(message.mid))
        key_mid_multicast = hash(str(defines.ALL_COAP_NODES) + str(port) + str(message.mid))
        key_token = hash(str(host) + str(port) + str(message.token))
        key_token_multicast = hash(str(defines.ALL_COAP_NODES) + str(port) + str(message.token))
        if key_mid in self._transactions.keys():
            transaction = self._transactions[key_mid]
        elif key_token in self._transactions_token:
            transaction = self._transactions_token[key_token]
        elif key_mid_multicast in self._transactions.keys():
            transaction = self._transactions[key_mid_multicast]
        elif key_token_multicast in self._transactions_token:
            transaction = self._transactions_token[key_token_multicast]
        else:
            logger.warning("Un-Matched incoming empty message " + str(host) + ":" + str(port))
            return None

        if message.type == defines.Types["ACK"]:
            if not transaction.request.acknowledged:
                transaction.request.acknowledged = True
            elif not transaction.response.acknowledged:
                transaction.response.acknowledged = True
        elif message.type == defines.Types["RST"]:
            if not transaction.request.acknowledged:
                transaction.request.rejected = True
            elif not transaction.response.acknowledged:
                transaction.response.rejected = True

        if transaction.retransmit_stop is not None:
            transaction.retransmit_stop.set()

        return transaction

    def send_request(self, request):
        """

        :type request: Request
        :param request:
        """
        logger.debug("send_request - " + str(request))
        assert isinstance(request, Request)
        try:
            host, port = request.destination
        except AttributeError:
            return

        request.timestamp = time.time()
        transaction = Transaction(request=request, timestamp=request.timestamp)
        if transaction.request.type is None:
            transaction.request.type = defines.Types["CON"]

        if transaction.request.mid is None:
            transaction.request.mid = self._current_mid
            self._current_mid += 1 % 65535

        key_mid = hash(str(host) + str(port) + str(request.mid))
        self._transactions[key_mid] = transaction

        key_token = hash(str(host) + str(port) + str(request.token))
        self._transactions_token[key_token] = transaction

        return self._transactions[key_mid]

    def send_response(self, transaction):
        """

        :type transaction: Transaction
        :param transaction:
        """
        logger.debug("send_response - " + str(transaction.response))
        if transaction.response.type is None:
            if transaction.request.type == defines.Types["CON"] and not transaction.request.acknowledged:
                transaction.response.type = defines.Types["ACK"]
                transaction.response.mid = transaction.request.mid
                transaction.response.acknowledged = True
                transaction.completed = True
            elif transaction.request.type == defines.Types["NON"]:
                transaction.response.type = defines.Types["NON"]
            else:
                transaction.response.type = defines.Types["CON"]

        if transaction.response.mid is None:
            transaction.response.mid = self._current_mid
            self._current_mid += 1 % 65535
            try:
                host, port = transaction.response.destination
            except AttributeError:
                return
            key_mid = hash(str(host).lower() + str(port).lower() + str(transaction.response.mid).lower())
            self._transactions[key_mid] = transaction

        transaction.request.acknowledged = True
        return transaction

    def send_empty(self, transaction, related, message):
        """

        :param transaction:
        :type message: Message
        :param message:
        """
        logger.debug("send_empty - " + str(message))
        if transaction is None:
            try:
                host, port = message.destination
            except AttributeError:
                return
            key_mid = hash(str(host).lower() + str(port).lower() + str(message.mid).lower())
            key_token = hash(str(host).lower() + str(port).lower() + str(message.token).lower())
            if key_mid in self._transactions:
                transaction = self._transactions[key_mid]
                related = transaction.response
            elif key_token in self._transactions_token:
                transaction = self._transactions_token[key_token]
                related = transaction.response
            else:
                return message

        if message.type == defines.Types["ACK"]:
            if transaction.request == related:
                transaction.request.acknowledged = True
                transaction.completed = True
                message._mid = transaction.request.mid
                message.code = 0
                message.token = transaction.request.token
                message.destination = transaction.request.source
            elif transaction.response == related:
                transaction.response.acknowledged = True
                transaction.completed = True
                message._mid = transaction.response.mid
                message.code = 0
                message.token = transaction.response.token
                message.destination = transaction.response.source

        elif message.type == defines.Types["RST"]:
            if transaction.request == related:
                transaction.request.rejected = True
                message._mid = transaction.request.mid
                message.code = 0
                message.token = transaction.request.token
                message.destination = transaction.request.source
            elif transaction.response == related:
                transaction.response.rejected = True
                transaction.completed = True
                message._mid = transaction.response.mid
                message.code = 0
                message.token = transaction.response.token
                message.destination = transaction.response.source
        return message

<commit_msg>Refactor repeated code<commit_after>import logging
import random
import time
from coapthon.messages.message import Message
from coapthon import defines
from coapthon.messages.request import Request
from coapthon.transaction import Transaction

logger = logging.getLogger(__name__)

def str_append_hash(*args):
    """ Convert each argument to a lower case string, appended, then hash """
    ret_hash = ""
    for i in args:
        ret_hash += str(i).lower()

    return hash(ret_hash)

class MessageLayer(object):
    def __init__(self, starting_mid):
        self._transactions = {}
        self._transactions_token = {}
        if starting_mid is not None:
            self._current_mid = starting_mid
        else:
            self._current_mid = random.randint(1, 1000)

    def purge(self):
        for k in self._transactions.keys():
            now = time.time()
            transaction = self._transactions[k]
            if transaction.timestamp + defines.EXCHANGE_LIFETIME < now:
                logger.debug("Delete transaction")
                del self._transactions[k]
        for k in self._transactions_token.keys():
            now = time.time()
            transaction = self._transactions_token[k]
            if transaction.timestamp + defines.EXCHANGE_LIFETIME < now:
                logger.debug("Delete transaction")
                del self._transactions_token[k]

    def receive_request(self, request):
        """

        :type request: Request
        :param request: the incoming request
        :rtype : Transaction
        """
        logger.debug("receive_request - " + str(request))
        try:
            host, port = request.source
        except AttributeError:
            return
        key_mid = str_append_hash(host, port, request.mid)
        key_token = str_append_hash(host, port, request.token)

        if key_mid in self._transactions.keys():
            # Duplicated
            self._transactions[key_mid].request.duplicated = True
            transaction = self._transactions[key_mid]
        else:
            request.timestamp = time.time()
            transaction = Transaction(request=request, timestamp=request.timestamp)
            with transaction:
                self._transactions[key_mid] = transaction
                self._transactions_token[key_token] = transaction
        return transaction

    def receive_response(self, response):
        """

        :type response: Response
        :param response:
        :rtype : Transaction
        """
        logger.debug("receive_response - " + str(response))
        try:
            host, port = response.source
        except AttributeError:
            return
        key_mid = str_append_hash(host, port, response.mid)
        key_mid_multicast = str_append_hash(defines.ALL_COAP_NODES, port, response.mid)
        key_token = str_append_hash(host, port, response.token)
        key_token_multicast = str_append_hash(defines.ALL_COAP_NODES, port, response.token)
        if key_mid in self._transactions.keys():
            transaction = self._transactions[key_mid]
        elif key_token in self._transactions_token:
            transaction = self._transactions_token[key_token]
        elif key_mid_multicast in self._transactions.keys():
            transaction = self._transactions[key_mid_multicast]
        elif key_token_multicast in self._transactions_token:
            transaction = self._transactions_token[key_token_multicast]
        else:
            logger.warning("Un-Matched incoming response message " + str(host) + ":" + str(port))
            return None, False
        send_ack = False
        if response.type == defines.Types["CON"]:
            send_ack = True

        transaction.request.acknowledged = True
        transaction.completed = True
        transaction.response = response
        if transaction.retransmit_stop is not None:
            transaction.retransmit_stop.set()
        return transaction, send_ack

    def receive_empty(self, message):
        """

        :type message: Message
        :param message:
        :rtype : Transaction
        """
        logger.debug("receive_empty - " + str(message))
        try:
            host, port = message.source
        except AttributeError:
            return
        key_mid = str_append_hash(host, port, message.mid)
        key_mid_multicast = str_append_hash(defines.ALL_COAP_NODES, port, message.mid)
        key_token = str_append_hash(host, port, message.token)
        key_token_multicast = str_append_hash(defines.ALL_COAP_NODES, port, message.token)
        if key_mid in self._transactions.keys():
            transaction = self._transactions[key_mid]
        elif key_token in self._transactions_token:
            transaction = self._transactions_token[key_token]
        elif key_mid_multicast in self._transactions.keys():
            transaction = self._transactions[key_mid_multicast]
        elif key_token_multicast in self._transactions_token:
            transaction = self._transactions_token[key_token_multicast]
        else:
            logger.warning("Un-Matched incoming empty message " + str(host) + ":" + str(port))
            return None

        if message.type == defines.Types["ACK"]:
            if not transaction.request.acknowledged:
                transaction.request.acknowledged = True
            elif not transaction.response.acknowledged:
                transaction.response.acknowledged = True
        elif message.type == defines.Types["RST"]:
            if not transaction.request.acknowledged:
                transaction.request.rejected = True
            elif not transaction.response.acknowledged:
                transaction.response.rejected = True

        if transaction.retransmit_stop is not None:
            transaction.retransmit_stop.set()

        return transaction

    def send_request(self, request):
        """

        :type request: Request
        :param request:
        """
        logger.debug("send_request - " + str(request))
        assert isinstance(request, Request)
        try:
            host, port = request.destination
        except AttributeError:
            return

        request.timestamp = time.time()
        transaction = Transaction(request=request, timestamp=request.timestamp)
        if transaction.request.type is None:
            transaction.request.type = defines.Types["CON"]

        if transaction.request.mid is None:
            transaction.request.mid = self._current_mid
            self._current_mid += 1 % 65535

        key_mid = str_append_hash(host, port, request.mid)
        self._transactions[key_mid] = transaction

        key_token = str_append_hash(host, port, request.token)
        self._transactions_token[key_token] = transaction

        return self._transactions[key_mid]

    def send_response(self, transaction):
        """

        :type transaction: Transaction
        :param transaction:
        """
        logger.debug("send_response - " + str(transaction.response))
        if transaction.response.type is None:
            if transaction.request.type == defines.Types["CON"] and not transaction.request.acknowledged:
                transaction.response.type = defines.Types["ACK"]
                transaction.response.mid = transaction.request.mid
                transaction.response.acknowledged = True
                transaction.completed = True
            elif transaction.request.type == defines.Types["NON"]:
                transaction.response.type = defines.Types["NON"]
            else:
                transaction.response.type = defines.Types["CON"]

        if transaction.response.mid is None:
            transaction.response.mid = self._current_mid
            self._current_mid += 1 % 65535
            try:
                host, port = transaction.response.destination
            except AttributeError:
                return
            key_mid = str_append_hash(host, port, transaction.response.mid)
            self._transactions[key_mid] = transaction

        transaction.request.acknowledged = True
        return transaction

    def send_empty(self, transaction, related, message):
        """

        :param transaction:
        :type message: Message
        :param message:
        """
        logger.debug("send_empty - " + str(message))
        if transaction is None:
            try:
                host, port = message.destination
            except AttributeError:
                return
            key_mid = str_append_hash(host, port, message.mid)
            key_token = str_append_hash(host, port, message.token)
            if key_mid in self._transactions:
                transaction = self._transactions[key_mid]
                related = transaction.response
            elif key_token in self._transactions_token:
                transaction = self._transactions_token[key_token]
                related = transaction.response
            else:
                return message

        if message.type == defines.Types["ACK"]:
            if transaction.request == related:
                transaction.request.acknowledged = True
                transaction.completed = True
                message._mid = transaction.request.mid
                message.code = 0
                message.token = transaction.request.token
                message.destination = transaction.request.source
            elif transaction.response == related:
                transaction.response.acknowledged = True
                transaction.completed = True
                message._mid = transaction.response.mid
                message.code = 0
                message.token = transaction.response.token
                message.destination = transaction.response.source

        elif message.type == defines.Types["RST"]:
            if transaction.request == related:
                transaction.request.rejected = True
                message._mid = transaction.request.mid
                message.code = 0
                message.token = transaction.request.token
                message.destination = transaction.request.source
            elif transaction.response == related:
                transaction.response.rejected = True
                transaction.completed = True
                message._mid = transaction.response.mid
                message.code = 0
                message.token = transaction.response.token
                message.destination = transaction.response.source
        return message

<|endoftext|>


<commit_before>from __future__ import division

from chainercv.datasets import CamVidDataset
import numpy as np

n_class = 12
d = CamVidDataset(mode='train')

n_cls_pixels = [0 for _ in range(n_class)]
n_img_pixels = [0 for _ in range(n_class)]

for img, lbl in d:
    for cls_i in np.unique(lbl):
        n_cls_pixels[cls_i] += np.sum(lbl == cls_i)
        n_img_pixels[cls_i] += lbl.size
freq = []
for n_cls_pixel, n_img_pixel in zip(n_cls_pixels, n_img_pixels):
    freq.append(n_cls_pixel / n_img_pixel)
freq = np.array(freq)
median_freq = np.median(freq)

np.save('class_weight', median_freq / freq)
<commit_msg>fix calc_weight<commit_after>from __future__ import division

import numpy as np

from chainercv.datasets import CamVidDataset

n_class = 12
dataset = CamVidDataset(split='train')

n_cls_pixels = [0 for _ in range(n_class)]
n_img_pixels = [0 for _ in range(n_class)]

for img, lbl in dataset:
    for cls_i in np.unique(lbl):
        n_cls_pixels[cls_i] += np.sum(lbl == cls_i)
        n_img_pixels[cls_i] += lbl.size
freq = []
for n_cls_pixel, n_img_pixel in zip(n_cls_pixels, n_img_pixels):
    freq.append(n_cls_pixel / n_img_pixel)
freq = np.array(freq)
median_freq = np.median(freq)

np.save('class_weight', median_freq / freq)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import

import multiprocessing
import os
import shutil
import time

import hglib

from cli_common.command import run_check
from cli_common.log import get_logger
from cli_common.mercurial import hg_run
from static_analysis_bot import CLANG_FORMAT
from static_analysis_bot import CLANG_TIDY
from static_analysis_bot import COVERAGE
from static_analysis_bot import COVERITY
from static_analysis_bot import INFER
from static_analysis_bot import MOZLINT
from static_analysis_bot import AnalysisException
from static_analysis_bot import stats
from static_analysis_bot.clang import setup as setup_clang
from static_analysis_bot.clang.format import ClangFormat
from static_analysis_bot.clang.tidy import ClangTidy
from static_analysis_bot.config import REPO_TRY
from static_analysis_bot.config import REPO_UNIFIED
from static_analysis_bot.config import SOURCE_TRY
from static_analysis_bot.config import Publication
from static_analysis_bot.config import settings
from static_analysis_bot.coverage import Coverage
from static_analysis_bot.coverity import setup as setup_coverity
from static_analysis_bot.coverity.coverity import Coverity
from static_analysis_bot.infer import setup as setup_infer
from static_analysis_bot.infer.infer import Infer
from static_analysis_bot.lint import MozLint

logger = get_logger(__name__)

class LocalWorkflow(object):
    '''
    Run analyers in current task
    '''
    def __init__(self, parent, analyzers, index_service):
        self.parent = parent
        assert isinstance(analyzers, list)
        assert len(analyzers) > 0, \
            'No analyzers specified, will not run.'
        self.analyzers = analyzers
        assert 'MOZCONFIG' in os.environ, \
            'Missing MOZCONFIG in environment'

        # Use TC services client
        self.index_service = index_service

    @stats.api.timed('runtime.clone')
    def clone(self, revision):
        '''
        Clone mozilla-unified
        '''
        logger.info('Clone mozilla unified', dir=settings.repo_dir, shared=settings.repo_shared_dir)
        if settings.source == SOURCE_TRY:
            # Clone Try using the target revision
            assert revision.mercurial_revision is not None, 'Missing try mercurial revision'
            cmd = hglib.util.cmdbuilder('robustcheckout',
                                        REPO_TRY,
                                        settings.repo_dir,
                                        revision=revision.mercurial_revision,
                                        upstream=REPO_UNIFIED,
                                        purge=True,
                                        sharebase=settings.repo_shared_dir)

        else:
            # Clone Mozilla unified up to central bookmark
            cmd = hglib.util.cmdbuilder('robustcheckout',
                                        REPO_UNIFIED,
                                        settings.repo_dir,
                                        purge=True,
                                        sharebase=settings.repo_shared_dir,
                                        branch=b'central')
        start = time.time()
        try:
            out = hg_run(cmd)
        except hglib.error.CommandError as e:
            logger.error('Mercurial clone failure', error=str(e))
            raise Exception('Mercurial clone failed with exit code {}'.format(e.ret))
        logger.info('Clone finished', time=(time.time() - start), out=out)

    def open_repository(self):
        '''
        As clone is run in a separate process, the bot needs to load a mercurial
        client in the main process
        '''

        # Open new hg client
        client = hglib.open(settings.repo_dir)

        # Attach logger
        client.setcbout(lambda msg: logger.info('Mercurial out={}'.format(msg.decode('utf-8'))))
        client.setcberr(lambda msg: logger.info('Mercurial err={}'.format(msg.decode('utf-8'))))

        # Store MC top revision after robustcheckout
        self.top_revision = client.log('reverse(public())', limit=1)[0].node
        logger.info('Mozilla unified top revision', revision=self.top_revision)

        # Mark local clone is present
        settings.has_local_clone = True

        return client

    def run(self, revision):
        '''
        Run the local static analysis workflow:
         * Pull revision from review
         * Checkout revision
         * Run static analyzers
        '''
        analyzers = []

        # Add log to find Taskcluster task in papertrail
        logger.info(
            'New static analysis',
            taskcluster_task=settings.taskcluster.task_id,
            taskcluster_run=settings.taskcluster.run_id,
            channel=settings.app_channel,
            publication=settings.publication.name,
            revision=str(revision),
        )
        stats.api.event(
            title='Static analysis on {} for {}'.format(settings.app_channel, revision),
            text='Task {} #{}'.format(settings.taskcluster.task_id, settings.taskcluster.run_id),
        )
        stats.api.increment('analysis')

        with stats.api.timer('runtime.mercurial'):
            try:
                # Clone in a controllable process
                # and kill this new process if it exceeds the maximum allowed runtime
                clone = multiprocessing.Process(target=self.clone, args=(revision, ))
                clone.start()
                clone.join(settings.max_clone_runtime)
                if clone.is_alive():
                    logger.error('Clone watchdog expired, stopping immediately')

                    # Kill the clone process
                    clone.terminate()

                    # Stop workflow
                    raise AnalysisException('watchdog', 'Clone watchdog expired')

                # Open a mercurial client in main process
                self.hg = self.open_repository()

                # Start by cloning the mercurial repository
                self.parent.index(revision, state='cloned')

                # Force cleanup to reset top of MU
                # otherwise previous pull are there
                self.hg.update(rev=self.top_revision, clean=True)
                logger.info('Set repo back to Mozilla unified top', rev=self.hg.identify())
            except hglib.error.CommandError as e:
                raise AnalysisException('mercurial', str(e))

            # Load and analyze revision patch
            revision.load(self.hg)
            revision.analyze_patch()

        with stats.api.timer('runtime.mach'):
            # Only run mach if revision has any C/C++ or Java files
            if revision.has_clang_files:
                self.do_build_setup()

                # Download clang build from Taskcluster
                # Use new clang-tidy paths, https://bugzilla.mozilla.org/show_bug.cgi?id=1495641
                logger.info('Setup Taskcluster clang build...')
                setup_clang(repository='mozilla-inbound', revision='revision.874a07fdb045b725edc2aaa656a8620ff439ec10')

                # Use clang-tidy & clang-format
                if CLANG_TIDY in self.analyzers:
                    analyzers.append(ClangTidy)
                else:
                    logger.info('Skip clang-tidy')
                if CLANG_FORMAT in self.analyzers:
                    analyzers.append(ClangFormat)
                else:
                    logger.info('Skip clang-format')

                # Run Coverity Scan
                if COVERITY in self.analyzers:
                    logger.info('Setup Taskcluster coverity build...')
                    try:
                        setup_coverity(self.index_service)
                        analyzers.append(Coverity)
                    except Exception as e:
                        logger.error('Coverity setup failed, skipping analyzer.', error=str(e))
                else:
                    logger.info('Skip Coverity')

                if COVERAGE in self.analyzers:
                    analyzers.append(Coverage)
                else:
                    logger.info('Skip coverage analysis')

            if revision.has_infer_files:
                if INFER in self.analyzers:
                    analyzers.append(Infer)
                    logger.info('Setup Taskcluster infer build...')
                    setup_infer(self.index_service)
                else:
                    logger.info('Skip infer')

            if not (revision.has_clang_files or revision.has_infer_files):
                logger.info('No clang or java files detected, skipping mach, infer and clang-*')

            # Setup python environment
            logger.info('Mach lint setup...')
            cmd = ['gecko-env', './mach', 'lint', '--list']
            with stats.api.timer('runtime.mach.lint'):
                out = run_check(cmd, cwd=settings.repo_dir)
            if 'error: problem with lint setup' in out.decode('utf-8'):
                raise AnalysisException('mach', 'Mach lint setup failed')

            # Always use mozlint
            if MOZLINT in self.analyzers:
                analyzers.append(MozLint)
            else:
                logger.info('Skip mozlint')

        if not analyzers:
            logger.error('No analyzers to use on revision')
            return

        self.parent.index(revision, state='analyzing')
        with stats.api.timer('runtime.issues'):
            # Detect initial issues
            if settings.publication == Publication.BEFORE_AFTER:
                before_patch = self.detect_issues(analyzers, revision, True)
                logger.info('Detected {} issue(s) before patch'.format(len(before_patch)))
                stats.api.increment('analysis.issues.before', len(before_patch))
                revision.reset()

            # Apply patch
            revision.apply(self.hg)

            if settings.publication == Publication.BEFORE_AFTER and revision.has_clang_files \
                    and (revision.has_clang_header_files or revision.has_idl_files):
                self.do_build_setup()

            # Detect new issues
            issues = self.detect_issues(analyzers, revision)
            logger.info('Detected {} issue(s) after patch'.format(len(issues)))
            stats.api.increment('analysis.issues.after', len(issues))

            # Mark newly found issues
            if settings.publication == Publication.BEFORE_AFTER:
                for issue in issues:
                    issue.is_new = issue not in before_patch

        # Avoid duplicates
        # but still output a list to be compatible with LocalWorkflow
        return list(set(issues))

    @stats.api.timer('runtime.mach.setup')
    def do_build_setup(self):
        # Mach pre-setup with mozconfig
        try:
            logger.info('Mach delete any existing obj dir')
            obj_dir = os.path.join(settings.repo_dir, 'obj-x86_64-pc-linux-gnu')
            if (os.path.exists(obj_dir)):
                shutil.rmtree(obj_dir)

            logger.info('Mach configure...')
            with stats.api.timer('runtime.mach.configure'):
                run_check(['gecko-env', './mach', 'configure'], cwd=settings.repo_dir)

            logger.info('Mach compile db...')
            with stats.api.timer('runtime.mach.build-backend'):
                run_check(['gecko-env', './mach', 'build-backend', '--backend=CompileDB'], cwd=settings.repo_dir)

            logger.info('Mach pre-export...')
            with stats.api.timer('runtime.mach.pre-export'):
                run_check(['gecko-env', './mach', 'build', 'pre-export'], cwd=settings.repo_dir)

            logger.info('Mach export...')
            with stats.api.timer('runtime.mach.export'):
                run_check(['gecko-env', './mach', 'build', 'export'], cwd=settings.repo_dir)
        except Exception as e:
            raise AnalysisException('mach', str(e))

    def detect_issues(self, analyzers, revision, is_before=False):
        '''
        Detect issues for this revision
        '''
        issues = []
        for analyzer_class in analyzers:
            # Build analyzer
            analyzer = analyzer_class()
            analyzer_issues = []
            try:
                if is_before:
                    if analyzer.can_run_before_patch():
                        # Run analyzer on revision and store generated issues
                        logger.info('Run {} with no patch applied'.format(analyzer_class.__name__))
                        analyzer_issues = analyzer.run(revision)
                    else:
                        logger.info('Skipped running {} with no patch applied'.format(analyzer_class.__name__))
                        continue
                else:
                    logger.info('Run {}'.format(analyzer_class.__name__))
                    analyzer_issues = analyzer.run(revision)

            except AnalysisException as ex:
                # Log the error to Sentry
                logger.error('Analyzer thrown exceptions during runtime', analyzer=analyzer_class.__name__, exception=ex)

            finally:
                # Clean up any uncommitted changes left behind by this analyzer.
                self.hg.revert(settings.repo_dir.encode('utf-8'), all=True)
                logger.info('Reverted all uncommitted changes in repo', rev=self.hg.identify())

                # Compute line hashes now before anything else changes the code.
                for issue in analyzer_issues:
                    issue.build_lines_hash()

                issues += analyzer_issues

        return issues
<commit_msg>staticnalysis/bot: Update clang-tidy setup revision to match clang 8 (#2032)<commit_after># -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import

import multiprocessing
import os
import shutil
import time

import hglib

from cli_common.command import run_check
from cli_common.log import get_logger
from cli_common.mercurial import hg_run
from static_analysis_bot import CLANG_FORMAT
from static_analysis_bot import CLANG_TIDY
from static_analysis_bot import COVERAGE
from static_analysis_bot import COVERITY
from static_analysis_bot import INFER
from static_analysis_bot import MOZLINT
from static_analysis_bot import AnalysisException
from static_analysis_bot import stats
from static_analysis_bot.clang import setup as setup_clang
from static_analysis_bot.clang.format import ClangFormat
from static_analysis_bot.clang.tidy import ClangTidy
from static_analysis_bot.config import REPO_TRY
from static_analysis_bot.config import REPO_UNIFIED
from static_analysis_bot.config import SOURCE_TRY
from static_analysis_bot.config import Publication
from static_analysis_bot.config import settings
from static_analysis_bot.coverage import Coverage
from static_analysis_bot.coverity import setup as setup_coverity
from static_analysis_bot.coverity.coverity import Coverity
from static_analysis_bot.infer import setup as setup_infer
from static_analysis_bot.infer.infer import Infer
from static_analysis_bot.lint import MozLint

logger = get_logger(__name__)

class LocalWorkflow(object):
    '''
    Run analyers in current task
    '''
    def __init__(self, parent, analyzers, index_service):
        self.parent = parent
        assert isinstance(analyzers, list)
        assert len(analyzers) > 0, \
            'No analyzers specified, will not run.'
        self.analyzers = analyzers
        assert 'MOZCONFIG' in os.environ, \
            'Missing MOZCONFIG in environment'

        # Use TC services client
        self.index_service = index_service

    @stats.api.timed('runtime.clone')
    def clone(self, revision):
        '''
        Clone mozilla-unified
        '''
        logger.info('Clone mozilla unified', dir=settings.repo_dir, shared=settings.repo_shared_dir)
        if settings.source == SOURCE_TRY:
            # Clone Try using the target revision
            assert revision.mercurial_revision is not None, 'Missing try mercurial revision'
            cmd = hglib.util.cmdbuilder('robustcheckout',
                                        REPO_TRY,
                                        settings.repo_dir,
                                        revision=revision.mercurial_revision,
                                        upstream=REPO_UNIFIED,
                                        purge=True,
                                        sharebase=settings.repo_shared_dir)

        else:
            # Clone Mozilla unified up to central bookmark
            cmd = hglib.util.cmdbuilder('robustcheckout',
                                        REPO_UNIFIED,
                                        settings.repo_dir,
                                        purge=True,
                                        sharebase=settings.repo_shared_dir,
                                        branch=b'central')
        start = time.time()
        try:
            out = hg_run(cmd)
        except hglib.error.CommandError as e:
            logger.error('Mercurial clone failure', error=str(e))
            raise Exception('Mercurial clone failed with exit code {}'.format(e.ret))
        logger.info('Clone finished', time=(time.time() - start), out=out)

    def open_repository(self):
        '''
        As clone is run in a separate process, the bot needs to load a mercurial
        client in the main process
        '''

        # Open new hg client
        client = hglib.open(settings.repo_dir)

        # Attach logger
        client.setcbout(lambda msg: logger.info('Mercurial out={}'.format(msg.decode('utf-8'))))
        client.setcberr(lambda msg: logger.info('Mercurial err={}'.format(msg.decode('utf-8'))))

        # Store MC top revision after robustcheckout
        self.top_revision = client.log('reverse(public())', limit=1)[0].node
        logger.info('Mozilla unified top revision', revision=self.top_revision)

        # Mark local clone is present
        settings.has_local_clone = True

        return client

    def run(self, revision):
        '''
        Run the local static analysis workflow:
         * Pull revision from review
         * Checkout revision
         * Run static analyzers
        '''
        analyzers = []

        # Add log to find Taskcluster task in papertrail
        logger.info(
            'New static analysis',
            taskcluster_task=settings.taskcluster.task_id,
            taskcluster_run=settings.taskcluster.run_id,
            channel=settings.app_channel,
            publication=settings.publication.name,
            revision=str(revision),
        )
        stats.api.event(
            title='Static analysis on {} for {}'.format(settings.app_channel, revision),
            text='Task {} #{}'.format(settings.taskcluster.task_id, settings.taskcluster.run_id),
        )
        stats.api.increment('analysis')

        with stats.api.timer('runtime.mercurial'):
            try:
                # Clone in a controllable process
                # and kill this new process if it exceeds the maximum allowed runtime
                clone = multiprocessing.Process(target=self.clone, args=(revision, ))
                clone.start()
                clone.join(settings.max_clone_runtime)
                if clone.is_alive():
                    logger.error('Clone watchdog expired, stopping immediately')

                    # Kill the clone process
                    clone.terminate()

                    # Stop workflow
                    raise AnalysisException('watchdog', 'Clone watchdog expired')

                # Open a mercurial client in main process
                self.hg = self.open_repository()

                # Start by cloning the mercurial repository
                self.parent.index(revision, state='cloned')

                # Force cleanup to reset top of MU
                # otherwise previous pull are there
                self.hg.update(rev=self.top_revision, clean=True)
                logger.info('Set repo back to Mozilla unified top', rev=self.hg.identify())
            except hglib.error.CommandError as e:
                raise AnalysisException('mercurial', str(e))

            # Load and analyze revision patch
            revision.load(self.hg)
            revision.analyze_patch()

        with stats.api.timer('runtime.mach'):
            # Only run mach if revision has any C/C++ or Java files
            if revision.has_clang_files:
                self.do_build_setup()

                # Download clang build from Taskcluster
                # Use updated clang-tidy matching clang/llvm 8
                logger.info('Setup Taskcluster clang build...')
                setup_clang(repository='autoland', revision='revision.7d60a7fd2fac0ac7427666f2d857aeba128cc6d0')

                # Use clang-tidy & clang-format
                if CLANG_TIDY in self.analyzers:
                    analyzers.append(ClangTidy)
                else:
                    logger.info('Skip clang-tidy')
                if CLANG_FORMAT in self.analyzers:
                    analyzers.append(ClangFormat)
                else:
                    logger.info('Skip clang-format')

                # Run Coverity Scan
                if COVERITY in self.analyzers:
                    logger.info('Setup Taskcluster coverity build...')
                    try:
                        setup_coverity(self.index_service)
                        analyzers.append(Coverity)
                    except Exception as e:
                        logger.error('Coverity setup failed, skipping analyzer.', error=str(e))
                else:
                    logger.info('Skip Coverity')

                if COVERAGE in self.analyzers:
                    analyzers.append(Coverage)
                else:
                    logger.info('Skip coverage analysis')

            if revision.has_infer_files:
                if INFER in self.analyzers:
                    analyzers.append(Infer)
                    logger.info('Setup Taskcluster infer build...')
                    setup_infer(self.index_service)
                else:
                    logger.info('Skip infer')

            if not (revision.has_clang_files or revision.has_infer_files):
                logger.info('No clang or java files detected, skipping mach, infer and clang-*')

            # Setup python environment
            logger.info('Mach lint setup...')
            cmd = ['gecko-env', './mach', 'lint', '--list']
            with stats.api.timer('runtime.mach.lint'):
                out = run_check(cmd, cwd=settings.repo_dir)
            if 'error: problem with lint setup' in out.decode('utf-8'):
                raise AnalysisException('mach', 'Mach lint setup failed')

            # Always use mozlint
            if MOZLINT in self.analyzers:
                analyzers.append(MozLint)
            else:
                logger.info('Skip mozlint')

        if not analyzers:
            logger.error('No analyzers to use on revision')
            return

        self.parent.index(revision, state='analyzing')
        with stats.api.timer('runtime.issues'):
            # Detect initial issues
            if settings.publication == Publication.BEFORE_AFTER:
                before_patch = self.detect_issues(analyzers, revision, True)
                logger.info('Detected {} issue(s) before patch'.format(len(before_patch)))
                stats.api.increment('analysis.issues.before', len(before_patch))
                revision.reset()

            # Apply patch
            revision.apply(self.hg)

            if settings.publication == Publication.BEFORE_AFTER and revision.has_clang_files \
                    and (revision.has_clang_header_files or revision.has_idl_files):
                self.do_build_setup()

            # Detect new issues
            issues = self.detect_issues(analyzers, revision)
            logger.info('Detected {} issue(s) after patch'.format(len(issues)))
            stats.api.increment('analysis.issues.after', len(issues))

            # Mark newly found issues
            if settings.publication == Publication.BEFORE_AFTER:
                for issue in issues:
                    issue.is_new = issue not in before_patch

        # Avoid duplicates
        # but still output a list to be compatible with LocalWorkflow
        return list(set(issues))

    @stats.api.timer('runtime.mach.setup')
    def do_build_setup(self):
        # Mach pre-setup with mozconfig
        try:
            logger.info('Mach delete any existing obj dir')
            obj_dir = os.path.join(settings.repo_dir, 'obj-x86_64-pc-linux-gnu')
            if (os.path.exists(obj_dir)):
                shutil.rmtree(obj_dir)

            logger.info('Mach configure...')
            with stats.api.timer('runtime.mach.configure'):
                run_check(['gecko-env', './mach', 'configure'], cwd=settings.repo_dir)

            logger.info('Mach compile db...')
            with stats.api.timer('runtime.mach.build-backend'):
                run_check(['gecko-env', './mach', 'build-backend', '--backend=CompileDB'], cwd=settings.repo_dir)

            logger.info('Mach pre-export...')
            with stats.api.timer('runtime.mach.pre-export'):
                run_check(['gecko-env', './mach', 'build', 'pre-export'], cwd=settings.repo_dir)

            logger.info('Mach export...')
            with stats.api.timer('runtime.mach.export'):
                run_check(['gecko-env', './mach', 'build', 'export'], cwd=settings.repo_dir)
        except Exception as e:
            raise AnalysisException('mach', str(e))

    def detect_issues(self, analyzers, revision, is_before=False):
        '''
        Detect issues for this revision
        '''
        issues = []
        for analyzer_class in analyzers:
            # Build analyzer
            analyzer = analyzer_class()
            analyzer_issues = []
            try:
                if is_before:
                    if analyzer.can_run_before_patch():
                        # Run analyzer on revision and store generated issues
                        logger.info('Run {} with no patch applied'.format(analyzer_class.__name__))
                        analyzer_issues = analyzer.run(revision)
                    else:
                        logger.info('Skipped running {} with no patch applied'.format(analyzer_class.__name__))
                        continue
                else:
                    logger.info('Run {}'.format(analyzer_class.__name__))
                    analyzer_issues = analyzer.run(revision)

            except AnalysisException as ex:
                # Log the error to Sentry
                logger.error('Analyzer thrown exceptions during runtime', analyzer=analyzer_class.__name__, exception=ex)

            finally:
                # Clean up any uncommitted changes left behind by this analyzer.
                self.hg.revert(settings.repo_dir.encode('utf-8'), all=True)
                logger.info('Reverted all uncommitted changes in repo', rev=self.hg.identify())

                # Compute line hashes now before anything else changes the code.
                for issue in analyzer_issues:
                    issue.build_lines_hash()

                issues += analyzer_issues

        return issues
<|endoftext|>


<commit_before># Lint as: python3

# Copyright 2020 The Flax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for flax.examples.seq2seq.train."""

from absl.testing import absltest

from flax import nn
import train

from jax import random

class TrainTest(absltest.TestCase):

  def test_character_table(self):
    text = '410+19'
    enc_text = train.CTABLE.encode(text)
    dec_text = train.CTABLE.decode(enc_text)
    # The text is possibly padded with whitespace, but the trimmed output should
    # be equal to the input.
    self.assertEqual(text, dec_text.strip())

  def test_train_one_step(self):
    batch = train.get_batch(128)
    rng = random.PRNGKey(0)

    with nn.stochastic(rng):
      model = train.create_model()
      optimizer = train.create_optimizer(model, 0.003)
      optimizer, train_metrics = train.train_step(optimizer, batch)

    self.assertLessEqual(train_metrics['loss'], 5)
    self.assertGreaterEqual(train_metrics['accuracy'], 0)

if __name__ == '__main__':
  absltest.main()
<commit_msg>Fix tests in seq2seq<commit_after># Lint as: python3

# Copyright 2020 The Flax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for flax.examples.seq2seq.train."""

from absl.testing import absltest

from flax import nn
import train

import jax
from jax import random

jax.config.parse_flags_with_absl()

class TrainTest(absltest.TestCase):

  def test_character_table(self):
    text = '410+19'
    enc_text = train.CTABLE.encode(text)
    dec_text = train.CTABLE.decode(enc_text)
    # The text is possibly padded with whitespace, but the trimmed output should
    # be equal to the input.
    self.assertEqual(text, dec_text.strip())

  def test_train_one_step(self):
    batch = train.get_batch(128)
    rng = random.PRNGKey(0)

    with nn.stochastic(rng):
      model = train.create_model()
      optimizer = train.create_optimizer(model, 0.003)
      optimizer, train_metrics = train.train_step(optimizer, batch)

    self.assertLessEqual(train_metrics['loss'], 5)
    self.assertGreaterEqual(train_metrics['accuracy'], 0)

if __name__ == '__main__':
  absltest.main()
<|endoftext|>


<commit_before>from openerp import api,models,fields

class product_style(models.Model):
    _inherit = "product.style"

    name = fields.Char(translate=True)
    auto = fields.Boolean('Auto style', default=False)
    can_be_ordered = fields.Boolean('Can be ordered', default=True)

class product_template(models.Model):
    _inherit = 'product.template'

    @api.model
    def update_stock_status_all(self):
        self.search([]).update_stock_status()

    @api.one
    def update_stock_status(self):
        website_style_ids = []
        stock = self.qty_available
        state = self.state
        ribbon_backordered = self.env.ref("website_sale_stock_status.ribbon_backordered").id
        ribbon_discontinued = self.env.ref("website_sale_stock_status.ribbon_discontinued").id
        # 3 - unlink, 4 - link
        if stock > 0:
            website_style_ids.append( (3, ribbon_backordered, 0) )
            website_style_ids.append( (3, ribbon_discontinued, 0) )
        elif state in ['end', 'obsolete']:
            website_style_ids.append( (3, ribbon_backordered, 0) )
            website_style_ids.append( (4, ribbon_discontinued, 0) )
        else:
            website_style_ids.append( (4, ribbon_backordered, 0) )
            website_style_ids.append( (3, ribbon_discontinued, 0) )
        self.website_style_ids =  website_style_ids
<commit_msg>[FIX] optimise update_stock_status_all<commit_after>from openerp import api,models,fields

class product_style(models.Model):
    _inherit = "product.style"

    name = fields.Char(translate=True)
    auto = fields.Boolean('Auto style', default=False)
    can_be_ordered = fields.Boolean('Can be ordered', default=True)

class product_template(models.Model):
    _inherit = 'product.template'

    @api.model
    def update_stock_status_all(self):
        ribbon_backordered = self.env.ref("website_sale_stock_status.ribbon_backordered").id
        ribbon_discontinued = self.env.ref("website_sale_stock_status.ribbon_discontinued").id
        # 3 - unlink, 4 - link
        self.search([('qty_available', '>', 0)]).write({'website_style_ids':[(3, ribbon_backordered, 0), (3, ribbon_discontinued, 0)]})
        self.search([('qty_available', '=', 0), ('state', 'in', ['end', 'obsolete'])]).write({'website_style_ids':[(3, ribbon_backordered, 0), (4, ribbon_discontinued, 0)]})
        self.search([('qty_available', '=', 0), ('state', 'not in', ['end', 'obsolete'])]).write({'website_style_ids':[(4, ribbon_backordered, 0), (3, ribbon_discontinued, 0)]})
<|endoftext|>


<commit_before>#
# example_site/books/models.py
#

import logging

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse, NoReverseMatch

from dcolumn.common.model_mixins import (
    UserModelMixin, TimeModelMixin, StatusModelMixin, StatusModelManagerMixin)
from dcolumn.dcolumns.models import CollectionBase, CollectionBaseManagerBase

from .choices import Language

log = logging.getLogger('example_site.models')

#
# Promotion
#
class PromotionManager(StatusModelManagerMixin):

    def dynamic_column(self, active=True):
        """
        We need to return all choices even if some are inactive, because
        the store_relation field in DynamicColumn' is acive for this model.
        """
        return self.active(active=active)

class Promotion(UserModelMixin, TimeModelMixin, StatusModelMixin):
    name = models.CharField(
        verbose_name=_(u"Promotion's Name"), max_length=250,
        help_text=_(u"Enter the name of the promotion."))
    description = models.TextField(
        verbose_name=_(u"Description"),
        help_text=_(u"Enter a description of the book promotion."))
    start_date = models.DateTimeField(
        verbose_name=_(u"Start Date & Time"), null=True, blank=True,
        help_text=_(u"Enter the start date and time of this promotion."))
    end_date = models.DateTimeField(
        verbose_name=_(u"End Date & Time"), null=True, blank=True,
        help_text=_(u"Enter the end date and time of this promotion."))

    objects = PromotionManager()

    class Meta:
        ordering = ('-start_date', 'name',)
        verbose_name = _(u"Promotion")
        verbose_name_plural = _(u"Promotions")

    def save(self, *args, **kwargs):
        super(Promotion, self).save(*args, **kwargs)

    def __unicode__(self):
        return u"{}-{}".format(self.name, self.start_date.isoformat())

    def get_absolute_url(self):
        return reverse('promotion-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True

#
# Author
#
class AuthorManager(CollectionBaseManagerBase, StatusModelManagerMixin):

    def dynamic_column(self):
        return self.active()

class Author(CollectionBase):
    name = models.CharField(
        verbose_name=_("Author's Name"), max_length=250,
        help_text=_("Enter the name of the author."))

    objects = AuthorManager()

    class Meta:
        ordering = ('name',)
        verbose_name = _("Author")
        verbose_name_plural = _("Authors")

    def save(self, *args, **kwargs):
        super(Author, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('author-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True

#
# Publisher
#
class PublisherManager(CollectionBaseManagerBase, StatusModelManagerMixin):

    def dynamic_column(self):
        return self.active()

class Publisher(CollectionBase):
    name = models.CharField(
        verbose_name=_("Publisher's Name"), max_length=250,
        help_text=_("Enter the name of the publisher."))

    objects = PublisherManager()

    class Meta:
        verbose_name = _("Publisher")
        verbose_name_plural = _("Publishers")

    def save(self, *args, **kwargs):
        super(Publisher, self).save(*args, **kwargs)

    def __unicode__(self):
        return unicode("{}".format(self.name))

    def get_absolute_url(self):
        return reverse('publisher-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True

#
# Book
#
class BookManager(CollectionBaseManagerBase, StatusModelManagerMixin):

    def dynamic_column(self):
        return self.active()

class Book(CollectionBase):
    title = models.CharField(
        verbose_name=_("Title"), max_length=250,
        help_text=_("Enter a book title."))

    objects = BookManager()

    class Meta:
        verbose_name = _("Book")
        verbose_name_plural = _("Books")

    def save(self, *args, **kwargs):
        super(Book, self).save(*args, **kwargs)

    def __unicode__(self):
        return unicode("{}".format(self.title))

    def get_absolute_url(self):
        return reverse('book-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True
<commit_msg>Updated some example code to conform with changes in dcolumn.<commit_after>#
# example_site/books/models.py
#

import logging

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse, NoReverseMatch

from dcolumn.common.model_mixins import (
    UserModelMixin, TimeModelMixin, StatusModelMixin, StatusModelManagerMixin,
    BaseChoiceModelManager)
from dcolumn.dcolumns.models import CollectionBase, CollectionBaseManagerBase

from .choices import Language

log = logging.getLogger('example_site.models')

#
# Promotion
#
class PromotionManager(BaseChoiceModelManager, StatusModelManagerMixin):

    def dynamic_column(self, active=True):
        """
        We need to return all choices even if some are inactive, because
        the store_relation field in DynamicColumn' is acive for this model.
        """
        return self.active(active=active)

    def get_choice_map(self, field, active=True):
        return dict([(getattr(obj, field), obj.pk)
                     for obj in self.dynamic_column(active=active)])

class Promotion(UserModelMixin, TimeModelMixin, StatusModelMixin):
    name = models.CharField(
        verbose_name=_(u"Promotion's Name"), max_length=250,
        help_text=_(u"Enter the name of the promotion."))
    description = models.TextField(
        verbose_name=_(u"Description"),
        help_text=_(u"Enter a description of the book promotion."))
    start_date = models.DateTimeField(
        verbose_name=_(u"Start Date & Time"), null=True, blank=True,
        help_text=_(u"Enter the start date and time of this promotion."))
    end_date = models.DateTimeField(
        verbose_name=_(u"End Date & Time"), null=True, blank=True,
        help_text=_(u"Enter the end date and time of this promotion."))

    objects = PromotionManager()

    class Meta:
        ordering = ('-start_date', 'name',)
        verbose_name = _(u"Promotion")
        verbose_name_plural = _(u"Promotions")

    def save(self, *args, **kwargs):
        super(Promotion, self).save(*args, **kwargs)

    def __unicode__(self):
        return u"{}-{}".format(self.name, self.start_date.isoformat())

    def get_absolute_url(self):
        return reverse('promotion-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True

#
# Author
#
class AuthorManager(CollectionBaseManagerBase, StatusModelManagerMixin,
                    BaseChoiceModelManager):

    def dynamic_column(self, active=True):
        return self.active(active=active)

    def get_choice_map(self, field, active=True):
        return dict([(getattr(obj, field), obj.pk)
                     for obj in self.dynamic_column(active=active)])

class Author(CollectionBase):
    name = models.CharField(
        verbose_name=_("Author's Name"), max_length=250,
        help_text=_("Enter the name of the author."))

    objects = AuthorManager()

    class Meta:
        ordering = ('name',)
        verbose_name = _("Author")
        verbose_name_plural = _("Authors")

    def save(self, *args, **kwargs):
        super(Author, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('author-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True

#
# Publisher
#
class PublisherManager(CollectionBaseManagerBase, StatusModelManagerMixin,
                       BaseChoiceModelManager):

    def dynamic_column(self, active=True):
        return self.active(active=active)

    def get_choice_map(self, field, active=True):
        return dict([(getattr(obj, field), obj.pk)
                     for obj in self.dynamic_column(active=active)])

class Publisher(CollectionBase):
    name = models.CharField(
        verbose_name=_("Publisher's Name"), max_length=250,
        help_text=_("Enter the name of the publisher."))

    objects = PublisherManager()

    class Meta:
        verbose_name = _("Publisher")
        verbose_name_plural = _("Publishers")

    def save(self, *args, **kwargs):
        super(Publisher, self).save(*args, **kwargs)

    def __unicode__(self):
        return unicode("{}".format(self.name))

    def get_absolute_url(self):
        return reverse('publisher-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True

#
# Book
#
class BookManager(CollectionBaseManagerBase, StatusModelManagerMixin,
                  BaseChoiceModelManager):

    def dynamic_column(self, active=True):
        return self.active(active=active)

    def get_choice_map(self, field, active=True):
        return dict([(getattr(obj, field), obj.pk)
                     for obj in self.dynamic_column(active=active)])

class Book(CollectionBase):
    title = models.CharField(
        verbose_name=_("Title"), max_length=250,
        help_text=_("Enter a book title."))

    objects = BookManager()

    class Meta:
        verbose_name = _("Book")
        verbose_name_plural = _("Books")

    def save(self, *args, **kwargs):
        super(Book, self).save(*args, **kwargs)

    def __unicode__(self):
        return unicode("{}".format(self.title))

    def get_absolute_url(self):
        return reverse('book-detail', kwargs={u'pk': self.pk})

    def _detail_producer(self):
        return u'View Page'.format(self.get_absolute_url())
    _detail_producer.short_description = _(u"View Detail")
    _detail_producer.allow_tags = True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import abc
import os

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
from BrokerRequest import BrokerRequest

import src.utils as u

class EBrokerClient:
    __metaclass__ = abc.ABCMeta

    def __init__(self, executable, login, password):
        self._browser = self._create_browser(executable)
        self.login(login, password)

    @abc.abstractmethod
    def _create_browser(self, executable):
        pass

    def login(self, login, password):
            self._browser.get("https://www.fio.cz/e-broker/e-broker.cgi")
            assert "e-Broker: Login" in self._browser.title
            self._browser.find_element_by_name("LOGIN_USERNAME").send_keys(login)
            self._browser.find_element_by_name("LOGIN_PASSWORD").send_keys(password)
            self._browser.find_element_by_name("SUBMIT").click()
            assert "Portfolio/Stav" in self._browser.title

    def close(self):
        self._browser.close()

    def get_broker_requests(self):
        self._browser.get("https://www.fio.cz/e-broker/e-pokyny.cgi")
        assert "Pokyny" in self._browser.title

        requests = []
        #TODO - why? just use get_element on _browser
        html = self._browser.execute_script("return document.getElementById('pokyny_full_table').innerHTML")
        soup = BeautifulSoup(html)
        for tr in soup.find_all("tr"):
            tds = tr.find_all("td")
            if len(tds) >= 6:
                ticker = int(u.get_ticker_from_link(tds[1].contents[0]['href']))
                symbol = tds[1].contents[0].contents[0]
                price = u.str2float(tds[4].contents[0])
                amount = u.str2float(tds[5].contents[0])
                amount_satisfied = u.str2float(tds[6].contents[0])
                requests.append(BrokerRequest(ticker, price, amount, symbol, amount_satisfied))
        return requests

    def get_portfolio(self):
        self._browser.get("https://www.fio.cz/e-broker/e-portfolio.cgi")
        assert "Portfolio/Stav" in self._browser.title

        html = self._browser.execute_script("return document.getElementById('portfolio_table').innerHTML")
        soup = BeautifulSoup(html)
        trs = soup.find_all("tr")
        for i in range(2, len(trs) - 1):  # skip first two rows (table header) and the last one (summary)
            tds = trs[i].find_all("td")
            if len(tds) >= 4 :
                ticker = int(u.get_ticker_from_link(tds[2].contents[0]['href']))
                symbol = tds[2].contents[0].contents[0]
                amount = u.str2float(tds[3].contents[0])
                price = u.str2float(tds[4].contents[0].contents[0])
                print "%i %s %s %s %s" % (i, ticker, symbol, amount, price)

    def send_request(self, request):
        self._browser.get("https://www.fio.cz/e-broker/e-pokyn_univ.cgi?ceninaId=%i&smer=%i" % (request.ticker, request.req_type))
        #assert "???" in self._browser.title #TODO add assert
        self._browser.find_element_by_name("pocet").send_keys(request.amount)
        self._browser.find_element_by_name("cena").send_keys(request.price)
        valid_till_element = self._browser.find_element_by_name("platnyDo")
        valid_till_element.clear()
        valid_till_element.send_keys(u.shift_today(14))
        #switch to RMS
        self._browser.find_element_by_xpath('//select[@id="trh"]/option[@value="RMS"]').click()
        self._browser.find_element_by_id("easyClick").click()
        self._browser.find_element_by_id("buttonValidate").click()
        #assert "???" in self._browser.title #TODO add assert
        self._browser.find_element_by_xpath('//input[@value="Odeslat"]').click()
        #assert "???" in self._browser.title #TODO add assert

    def get_money(self):
        self._browser.get("https://www.fio.cz/e-broker/e-penize.cgi")
        assert u"Peníze" in self._browser.title

        money = []
        html = self._browser.find_element_by_id("penize_table").get_attribute("innerHTML")
        soup = BeautifulSoup(html)
        trs = soup.find_all("tr")
        for i in range(2, len(trs)):
            tds = trs[i].find_all("td")
            if len(tds) >= 9:
                currency = tds[2].contents[0]
                free_money = u.str2float(tds[8].contents[0])
                #TODO money object
                money.append([currency, free_money])
        return money

class PhantomJSClient(EBrokerClient):
    def _create_browser(self, phantom_exec):
        return webdriver.PhantomJS(phantom_exec)

class ChromeClient(EBrokerClient):
    def _create_browser(self, chrome_driver):
        opts = webdriver.ChromeOptions()
        prefs = {"download.default_directory" : os.getcwd()}
        opts.add_experimental_option("prefs", prefs)
        browser = webdriver.Chrome(executable_path=chrome_driver, chrome_options=opts)
        return browser
<commit_msg>Use same methods for parsing tables<commit_after># -*- coding: utf-8 -*-

import abc
import os

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
from BrokerRequest import BrokerRequest

import src.utils as u

class EBrokerClient:
    __metaclass__ = abc.ABCMeta

    def __init__(self, executable, login, password):
        self._browser = self._create_browser(executable)
        self.login(login, password)

    @abc.abstractmethod
    def _create_browser(self, executable):
        pass

    def login(self, login, password):
            self._browser.get("https://www.fio.cz/e-broker/e-broker.cgi")
            assert "e-Broker: Login" in self._browser.title
            self._browser.find_element_by_name("LOGIN_USERNAME").send_keys(login)
            self._browser.find_element_by_name("LOGIN_PASSWORD").send_keys(password)
            self._browser.find_element_by_name("SUBMIT").click()
            assert "Portfolio/Stav" in self._browser.title

    def close(self):
        self._browser.close()

    def get_broker_requests(self):
        self._browser.get("https://www.fio.cz/e-broker/e-pokyny.cgi")
        assert "Pokyny" in self._browser.title

        requests = []
        #html = self._browser.execute_script("return document.getElementById('pokyny_full_table').innerHTML")
        html = self._browser.find_element_by_id("pokyny_full_table").get_attribute("innerHTML")
        soup = BeautifulSoup(html)
        trs = soup.find_all("tr")
        for i in range(2, len(trs)):
            tds = trs[i].find_all("td")
            if len(tds) >= 6:
                ticker = int(u.get_ticker_from_link(tds[1].contents[0]['href']))
                symbol = tds[1].contents[0].contents[0]
                price = u.str2float(tds[4].contents[0])
                amount = u.str2float(tds[5].contents[0])
                amount_satisfied = u.str2float(tds[6].contents[0])
                requests.append(BrokerRequest(ticker, price, amount, symbol, amount_satisfied))
        return requests

    def get_portfolio(self):
        self._browser.get("https://www.fio.cz/e-broker/e-portfolio.cgi")
        assert "Portfolio/Stav" in self._browser.title

        html = self._browser.execute_script("return document.getElementById('portfolio_table').innerHTML")
        soup = BeautifulSoup(html)
        trs = soup.find_all("tr")
        for i in range(2, len(trs) - 1):  # skip first two rows (table header) and the last one (summary)
            tds = trs[i].find_all("td")
            if len(tds) >= 4 :
                ticker = int(u.get_ticker_from_link(tds[2].contents[0]['href']))
                symbol = tds[2].contents[0].contents[0]
                amount = u.str2float(tds[3].contents[0])
                price = u.str2float(tds[4].contents[0].contents[0])
                print "%i %s %s %s %s" % (i, ticker, symbol, amount, price)

    def send_request(self, request):
        self._browser.get("https://www.fio.cz/e-broker/e-pokyn_univ.cgi?ceninaId=%i&smer=%i" % (request.ticker, request.req_type))
        #assert "???" in self._browser.title #TODO add assert
        self._browser.find_element_by_name("pocet").send_keys(request.amount)
        self._browser.find_element_by_name("cena").send_keys(request.price)
        valid_till_element = self._browser.find_element_by_name("platnyDo")
        valid_till_element.clear()
        valid_till_element.send_keys(u.shift_today(14))
        #switch to RMS
        self._browser.find_element_by_xpath('//select[@id="trh"]/option[@value="RMS"]').click()
        self._browser.find_element_by_id("easyClick").click()
        self._browser.find_element_by_id("buttonValidate").click()
        #assert "???" in self._browser.title #TODO add assert
        self._browser.find_element_by_xpath('//input[@value="Odeslat"]').click()
        #assert "???" in self._browser.title #TODO add assert

    def get_money(self):
        self._browser.get("https://www.fio.cz/e-broker/e-penize.cgi")
        assert u"Peníze" in self._browser.title

        money = []
        html = self._browser.find_element_by_id("penize_table").get_attribute("innerHTML")
        soup = BeautifulSoup(html)
        trs = soup.find_all("tr")
        for i in range(2, len(trs)):
            tds = trs[i].find_all("td")
            if len(tds) >= 9:
                currency = tds[2].contents[0]
                free_money = u.str2float(tds[8].contents[0])
                #TODO money object
                money.append([currency, free_money])
        return money

class PhantomJSClient(EBrokerClient):
    def _create_browser(self, phantom_exec):
        return webdriver.PhantomJS(phantom_exec)

class ChromeClient(EBrokerClient):
    def _create_browser(self, chrome_driver):
        opts = webdriver.ChromeOptions()
        prefs = {"download.default_directory" : os.getcwd()}
        opts.add_experimental_option("prefs", prefs)
        browser = webdriver.Chrome(executable_path=chrome_driver, chrome_options=opts)
        return browser
<|endoftext|>


<commit_before>'''
Created on Jan 4, 2014

Licence: GNU AGPL

@author: Christian Holl
'''

import os
import http.client

class file_list_entry(object):
    file_name=''
    directory_name=''
    byte_size=-1
    attribute_Archive=False;
    attribute_Directly=False;
    attribute_Volume=False;
    attribute_System=False;
    attribute_Hidden=False;
    attribute_ReadOnly=False;
    date_human=()
    time_human=()
    time=0
    date=0
    def __init__(self, file_name, directory_name, size, attributes, date, time):
        self.file_name=file_name
        self.directory_name=directory_name
        self.byte_size=size

        attributes=int(attributes)
        self.attribute_Archive=  not not(attributes & 1<<5)
        self.attribute_Directly= not not(attributes & 1<<4)
        self.attribute_Volume=   not not(attributes & 1<<3)
        self.attribute_System=   not not(attributes & 1<<2)
        self.attribute_Hidden=   not not(attributes & 1<<1)
        self.attribute_ReadOnly= not not(attributes & 1<<0)

        time=int(time)
        self.time=time;
        self.time_human=(((time&(0x1F<<11))>>11),((time&(0x3F<<5))>>5),(time&(0x1F))*2)

        date=int(date)
        self.date=date;
        self.date_human=(((date&(0x3F<<9))>>9)+1980,((date&(0xF<<5))>>5),date&(0x1F))

class command(object):
    '''OPCODE - DIR DATE ADDR LENGTH DATA  REQUIRED FWVERSION (Bigger/Smaller)'''
    Get_file_list=                      (100,1,0,0,0,0,'1.00.03',True)
    Get_the_number_of_files=            (101,1,0,0,0,0,'1.00.00',True)
    Get_update_status=                  (102,0,0,0,0,0,'1.00.00',True)
    Get_SSID=                           (104,0,0,0,0,0,'1.00.00',True)
    Get_network_password=               (105,0,0,0,0,0,'1.00.00',True)
    Get_MAC_address=                    (106,0,0,0,0,0,'1.00.00',True)
    Set_browser_language=               (107,0,0,0,0,0,'1.00.00',True)
    Get_the_firmware_version=           (108,0,0,0,0,0,''       ,True)
    Get_the_control_image=              (109,0,0,0,0,0,'2.00.00',True)
    Get_Wireless_LAN_mode=              (110,0,0,0,0,0,'2.00.00',True)
    Set_Wireless_LAN_timeout_length=    (111,0,0,0,0,0,'2.00.00',True)
    Get_application_unique_information= (117,0,0,0,0,0,'2.00.00',True)
    Get_CID=                            (120,0,0,0,0,0,'1.00.03',True)
    Get_data_from_shared_memory=        (130,0,0,1,1,0,'2.00.00',True)
    Set_data_to_shared_memory=          (131,0,0,0,0,0,'2.00.00',True)
    Get_the_number_of_empty_sectors=    (140,0,0,0,0,0,'1.00.03',True)
    Enable_Photo_Share_mode=            (200,0,0,0,0,0,'2.00.00',True)
    Disable_Photo_Share_mode=           (201,0,0,0,0,0,'2.00.00',True)
    Get_Photo_Share_mode_status=        (202,0,0,0,0,0,'2.00.00',True)
    Get_SSID_for_Photo_Share_mode=      (203,0,0,0,0,0,'2.00.00',True)

class connection(object):
    '''
    classdocs
    '''

    list_directory=100
    fwversion=''
    host=0
    port=0
    timeout=0
    start_date=-1
    start_time=-1

    def __init__(self, host, port, timeout):
        '''
        Constructor
        '''
        self.host=host
        self.port=port
        self.timeout=timeout

    def send_command(self, opcode, directory='', date=-1, addr=-1, data=(), length=-1):
        if(len(opcode)==8):
            url="/command.cgi?op=" + str(opcode[0])

            if(opcode[1] and len(directory)==0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires directory")
                return(-1,'')
            elif(opcode[1]):
                url += '&DIR=' + directory

            if(opcode[2] and date<0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires date")
                return(-1,'')
            elif(opcode[2]):
                url += '&DATE=' + date

            if(opcode[3] and addr<0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires addr")
                return(-1,'')
            elif(opcode[3]):
                url += '&ADDR=' + str(addr)

            if(opcode[4] and length==0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires length")
                return(-1,'')
            elif(opcode[4]):
                url += '&LEN=' + str(length)

            if(opcode[5] and len(data)==0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires data")
                return(-1,'')
            elif(opcode[5]):
                url += '&DATA=' + data

            #Is it a firmware query?
            if(len(opcode[6])!=0):

                if(len(self.fwversion)==0):

                    (ret, ver)=self.send_command(command.Get_the_firmware_version)
                    if(not ret):
                        self.fwversion=ver.decode("utf-8")[9:]
                        print("Firmware is: ")
                        print(self.fwversion)
                    else:
                        if(ret!=-2):
                            print("ERROR: Could not determine firmware version!")
                        return(-1,'')

                if(opcode[7]): #must be bigger than or equal to firmware version
                    if(opcode[6]>self.fwversion):
                        print("ERROR Opcode " + str(opcode[0]) + " not supported in firmware!")
                        return(-1,'')
                else:
                    if(opcode[6]>self.fwversion):
                        print("ERROR Opcode " + str(opcode[0]) + " not supported in firmware!")
                        return(-1,'')

            #Connect
            connection = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)

            try:
                connection.request("GET", url)
                response=connection.getresponse();
                return (response.status!=200,response.read())
            except:
                return (-2,'')
                pass

    def get_file_list(self,directory):
        (ret,lst)=self.send_command(command.Get_file_list,directory=directory)

        if(ret==0):

            lines=lst.decode("utf-8").split("\r\n")
            if(lines[0]=="WLANSD_FILELIST"):
                lines=lines[1:-1] #skip headline, and current dir at the end
            else:
                return (0,())

            outlst=[]
            for file in lines:
                e=file.split(",")

                if(len(e)!=6):
                    print("Error file list entry has " +str(len(e)) +" entrie(s) instead of expected 6, skipping entry")
                    continue;
                #(file_name, directory_name, size, attributes, date, time):
                f=file_list_entry(e[1],e[0],int(e[2]),int(e[3]),int(e[4]),int(e[5]))
                outlst.append(f)

            return (0,outlst)
        else:
            return (1,())

    def download_file(self, remote_location, local_path='', local_file_name=''):
        conn = http.client.HTTPConnection(self.host)
        if(len(local_file_name)==0):
            local_file_name = remote_location.split('/')[-1]
        file_size=0

        #does folder exist?
        if(not os.access(local_path, os.R_OK)):
            return (2,0,'')

        #add / if it is not there already
        if(len(local_path)!=0 and local_path[-1]!='/'):
            local_path+='/'

        #combine path and file
        local_path+=local_file_name

        #does file exist already?
        if(os.path.isfile(local_path)):
            return (3,0,'')

        print("Downloading:" + local_file_name)
        #get the stuff from the FlashAir
        conn.request("GET", remote_location)
        download = conn.getresponse()

        if(download.status==200):
            file = open(local_path, 'wb')
            while True:
                buffer=download.read(1024*32)
                if not buffer:
                    break;
                file_size += len(buffer)
                file.write(buffer)
            file.close()
        download.close()
        print("done (%d bytes) " % file_size)
        return (int(download.status!=200), file_size,local_path)

    def download_file_list_entry(self, entry,local_path='', local_filename=''):
        (status,size,local_filename)=self.download_file(entry.directory_name + '/' + entry.file_name, local_path, local_filename)
        if(status):
            return(1)

        if(size!=entry.byte_size):
            print("Error Size does not match")
            os.remove(local_filename)
            return(2)

        return(0)

    def sync_folder_to_remote_folder(self,remote_path='',local_path='',extensions=['JPG']):
        #all extensions to upper case
        extensions=[x.upper() for x in extensions]

        #get list of remote files
        (status, outlist)=self.get_file_list(remote_path)
        if(not status and len(outlist)):
            if(not os.access(local_path, os.R_OK)):
                return 2
            for entry in outlist:
                if ((entry.file_name.split('.')[-1].upper() in extensions) or len(extensions)==0):
                    self.download_file_list_entry(entry, local_path)

    def sync_new_pictures_since_start(self,remote_path='',local_path='',extensions=['JPG']):
        last_file=''

        #all extensions to upper case
        extensions=[x.upper() for x in extensions]

        #get list of remote files
        (status, outlist)=self.get_file_list(remote_path)

        #determine latest file date and time

        if(self.start_date < 0):
            last_entry=None
            for entry in outlist:
                if(entry.date>self.start_date):
                    self.start_date=entry.date
                    self.start_time=entry.time
                    last_entry=entry
                elif (entry.date==self.start_date):
                    if(entry.time>self.start_time):
                        self.start_time=entry.time
                        last_entry=entry
            if(last_entry!=None):
                last_file=local_path+'/'+last_entry.file_name
                self.download_file_list_entry(last_entry, local_path) #download latest file

        if(not status and len(outlist)):
            if(not os.access(local_path, os.R_OK)):
                return ()
            for entry in outlist:
                if ((entry.file_name.split('.')[-1].upper() in extensions) or len(extensions)==0):
                    if(entry.date>=self.start_date):
                        if(entry.date>self.start_date or entry.time>self.start_time):
                            if(not self.download_file_list_entry(entry, local_path)):
                                last_file=local_path+'/'+entry.file_name
        return last_file
<commit_msg>Re-download files that have changed size<commit_after>'''
Created on Jan 4, 2014

Licence: GNU AGPL

@author: Christian Holl
'''

import os
import http.client

class file_list_entry(object):
    file_name=''
    directory_name=''
    byte_size=-1
    attribute_Archive=False;
    attribute_Directly=False;
    attribute_Volume=False;
    attribute_System=False;
    attribute_Hidden=False;
    attribute_ReadOnly=False;
    date_human=()
    time_human=()
    time=0
    date=0
    def __init__(self, file_name, directory_name, size, attributes, date, time):
        self.file_name=file_name
        self.directory_name=directory_name
        self.byte_size=size

        attributes=int(attributes)
        self.attribute_Archive=  not not(attributes & 1<<5)
        self.attribute_Directly= not not(attributes & 1<<4)
        self.attribute_Volume=   not not(attributes & 1<<3)
        self.attribute_System=   not not(attributes & 1<<2)
        self.attribute_Hidden=   not not(attributes & 1<<1)
        self.attribute_ReadOnly= not not(attributes & 1<<0)

        time=int(time)
        self.time=time;
        self.time_human=(((time&(0x1F<<11))>>11),((time&(0x3F<<5))>>5),(time&(0x1F))*2)

        date=int(date)
        self.date=date;
        self.date_human=(((date&(0x3F<<9))>>9)+1980,((date&(0xF<<5))>>5),date&(0x1F))

class command(object):
    '''OPCODE - DIR DATE ADDR LENGTH DATA  REQUIRED FWVERSION (Bigger/Smaller)'''
    Get_file_list=                      (100,1,0,0,0,0,'1.00.03',True)
    Get_the_number_of_files=            (101,1,0,0,0,0,'1.00.00',True)
    Get_update_status=                  (102,0,0,0,0,0,'1.00.00',True)
    Get_SSID=                           (104,0,0,0,0,0,'1.00.00',True)
    Get_network_password=               (105,0,0,0,0,0,'1.00.00',True)
    Get_MAC_address=                    (106,0,0,0,0,0,'1.00.00',True)
    Set_browser_language=               (107,0,0,0,0,0,'1.00.00',True)
    Get_the_firmware_version=           (108,0,0,0,0,0,''       ,True)
    Get_the_control_image=              (109,0,0,0,0,0,'2.00.00',True)
    Get_Wireless_LAN_mode=              (110,0,0,0,0,0,'2.00.00',True)
    Set_Wireless_LAN_timeout_length=    (111,0,0,0,0,0,'2.00.00',True)
    Get_application_unique_information= (117,0,0,0,0,0,'2.00.00',True)
    Get_CID=                            (120,0,0,0,0,0,'1.00.03',True)
    Get_data_from_shared_memory=        (130,0,0,1,1,0,'2.00.00',True)
    Set_data_to_shared_memory=          (131,0,0,0,0,0,'2.00.00',True)
    Get_the_number_of_empty_sectors=    (140,0,0,0,0,0,'1.00.03',True)
    Enable_Photo_Share_mode=            (200,0,0,0,0,0,'2.00.00',True)
    Disable_Photo_Share_mode=           (201,0,0,0,0,0,'2.00.00',True)
    Get_Photo_Share_mode_status=        (202,0,0,0,0,0,'2.00.00',True)
    Get_SSID_for_Photo_Share_mode=      (203,0,0,0,0,0,'2.00.00',True)

class connection(object):
    '''
    classdocs
    '''

    list_directory=100
    fwversion=''
    host=0
    port=0
    timeout=0
    start_date=-1
    start_time=-1

    def __init__(self, host, port, timeout):
        '''
        Constructor
        '''
        self.host=host
        self.port=port
        self.timeout=timeout

    def send_command(self, opcode, directory='', date=-1, addr=-1, data=(), length=-1):
        if(len(opcode)==8):
            url="/command.cgi?op=" + str(opcode[0])

            if(opcode[1] and len(directory)==0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires directory")
                return(-1,'')
            elif(opcode[1]):
                url += '&DIR=' + directory

            if(opcode[2] and date<0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires date")
                return(-1,'')
            elif(opcode[2]):
                url += '&DATE=' + date

            if(opcode[3] and addr<0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires addr")
                return(-1,'')
            elif(opcode[3]):
                url += '&ADDR=' + str(addr)

            if(opcode[4] and length==0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires length")
                return(-1,'')
            elif(opcode[4]):
                url += '&LEN=' + str(length)

            if(opcode[5] and len(data)==0 ):
                print("ERROR Oppcode " + str(opcode[0]) + " requires data")
                return(-1,'')
            elif(opcode[5]):
                url += '&DATA=' + data

            #Is it a firmware query?
            if(len(opcode[6])!=0):

                if(len(self.fwversion)==0):

                    (ret, ver)=self.send_command(command.Get_the_firmware_version)
                    if(not ret):
                        self.fwversion=ver.decode("utf-8")[9:]
                        print("Firmware is: ")
                        print(self.fwversion)
                    else:
                        if(ret!=-2):
                            print("ERROR: Could not determine firmware version!")
                        return(-1,'')

                if(opcode[7]): #must be bigger than or equal to firmware version
                    if(opcode[6]>self.fwversion):
                        print("ERROR Opcode " + str(opcode[0]) + " not supported in firmware!")
                        return(-1,'')
                else:
                    if(opcode[6]>self.fwversion):
                        print("ERROR Opcode " + str(opcode[0]) + " not supported in firmware!")
                        return(-1,'')

            #Connect
            connection = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)

            try:
                connection.request("GET", url)
                response=connection.getresponse();
                return (response.status!=200,response.read())
            except:
                return (-2,'')
                pass

    def get_file_list(self,directory):
        (ret,lst)=self.send_command(command.Get_file_list,directory=directory)

        if(ret==0):

            lines=lst.decode("utf-8").split("\r\n")
            if(lines[0]=="WLANSD_FILELIST"):
                lines=lines[1:-1] #skip headline, and current dir at the end
            else:
                return (0,())

            outlst=[]
            for file in lines:
                e=file.split(",")

                if(len(e)!=6):
                    print("Error file list entry has " +str(len(e)) +" entrie(s) instead of expected 6, skipping entry")
                    continue;
                #(file_name, directory_name, size, attributes, date, time):
                f=file_list_entry(e[1],e[0],int(e[2]),int(e[3]),int(e[4]),int(e[5]))
                outlst.append(f)

            return (0,outlst)
        else:
            return (1,())

    def download_file(self, entry, local_path='', local_file_name=''):
        remote_location = entry.directory_name + '/' + entry.file_name
        conn = http.client.HTTPConnection(self.host)
        if(len(local_file_name)==0):
            local_file_name = remote_location.split('/')[-1]
        file_size=0

        #does folder exist?
        if(not os.access(local_path, os.R_OK)):
            return (2,0,'')

        #add / if it is not there already
        if(len(local_path)!=0 and local_path[-1]!='/'):
            local_path+='/'

        #combine path and file
        local_path+=local_file_name

        print("Remote: %s (%d)" % (remote_location, entry.byte_size))

        #does file exist already?
        if(os.path.isfile(local_path) and
           entry.byte_size == os.path.getsize(local_path)):
            return (3,0,'')

        if(os.path.isfile(local_path)):
            print("Refreshing:" + local_file_name)
        else:
            print("Downloading:" + local_file_name)
        #get the stuff from the FlashAir
        conn.request("GET", remote_location)
        download = conn.getresponse()

        if(download.status==200):
            file = open(local_path, 'wb')
            while True:
                buffer=download.read(1024*32)
                if not buffer:
                    break;
                file_size += len(buffer)
                file.write(buffer)
            file.close()
        download.close()
        print("done (%d bytes) " % file_size)
        return (int(download.status!=200), file_size,local_path)

    def download_file_list_entry(self, entry,local_path='', local_filename=''):
        (status,size,local_filename)=self.download_file(entry, local_path, local_filename)
        if(status):
            return(1)

        if(size!=entry.byte_size):
            print("Error Size does not match")
            os.remove(local_filename)
            return(2)

        return(0)

    def sync_folder_to_remote_folder(self,remote_path='',local_path='',extensions=['JPG']):
        #all extensions to upper case
        extensions=[x.upper() for x in extensions]

        #get list of remote files
        (status, outlist)=self.get_file_list(remote_path)
        if(not status and len(outlist)):
            if(not os.access(local_path, os.R_OK)):
                return 2
            for entry in outlist:
                if ((entry.file_name.split('.')[-1].upper() in extensions) or len(extensions)==0):
                    self.download_file_list_entry(entry, local_path)

    def sync_new_pictures_since_start(self,remote_path='',local_path='',extensions=['JPG']):
        last_file=''

        #all extensions to upper case
        extensions=[x.upper() for x in extensions]

        #get list of remote files
        (status, outlist)=self.get_file_list(remote_path)

        #determine latest file date and time

        if(self.start_date < 0):
            last_entry=None
            for entry in outlist:
                if(entry.date>self.start_date):
                    self.start_date=entry.date
                    self.start_time=entry.time
                    last_entry=entry
                elif (entry.date==self.start_date):
                    if(entry.time>self.start_time):
                        self.start_time=entry.time
                        last_entry=entry
            if(last_entry!=None):
                last_file=local_path+'/'+last_entry.file_name
                self.download_file_list_entry(last_entry, local_path) #download latest file

        if(not status and len(outlist)):
            if(not os.access(local_path, os.R_OK)):
                return ()
            for entry in outlist:
                if ((entry.file_name.split('.')[-1].upper() in extensions) or len(extensions)==0):
                    if(entry.date>=self.start_date):
                        if(entry.date>self.start_date or entry.time>self.start_time):
                            if(not self.download_file_list_entry(entry, local_path)):
                                last_file=local_path+'/'+entry.file_name
        return last_file
<|endoftext|>


<commit_before><commit_msg>Improves regex handling<commit_after><|endoftext|>