<commit_before>"""
Simple "flat pages".

These get used for the static (non-automated) large chunks of content. Notice
that pages don't have any actual notion of where they live; instead, they're
positioned into the URL structure using the nav app.
"""

import os
import re

from copy import deepcopy

from django.conf import settings
from django.core import validators
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver

from markupfield.fields import MarkupField
from markupfield.markup import DEFAULT_MARKUP_TYPES

import cmarkgfm
from cmarkgfm.cmark import Options as cmarkgfmOptions

from cms.models import ContentManageable
from fastly.utils import purge_url

from .managers import PageQuerySet

DEFAULT_MARKUP_TYPE = getattr(settings, 'DEFAULT_MARKUP_TYPE', 'restructuredtext')

# Set options for cmarkgfm for "unsafe" renderer, see https://github.com/theacodes/cmarkgfm#advanced-usage
CMARKGFM_UNSAFE_OPTIONS = (
    cmarkgfmOptions.CMARK_OPT_UNSAFE
)

PAGE_PATH_RE = re.compile(r"""
    ^
    /?                      # We can optionally start with a /
    ([a-z0-9-\.]+)            # Then at least one path segment...
    (/[a-z0-9-\.]+)*        # And then possibly more "/whatever" segments
    /?                      # Possibly ending with a slash
    $
    """,
    re.X
)

is_valid_page_path = validators.RegexValidator(
    regex=PAGE_PATH_RE,
    message=(
        'Please enter a valid URL segment, e.g. "foo" or "foo/bar". '
        'Only lowercase letters, numbers, hyphens and periods are allowed.'
    ),
)

RENDERERS = deepcopy(DEFAULT_MARKUP_TYPES)
for i, renderer in enumerate(RENDERERS):
    if renderer[0] == 'markdown':
        markdown_index = i

RENDERERS[markdown_index] = (
    'markdown',
    cmarkgfm.github_flavored_markdown_to_html,
    'Markdown'
)

RENDERERS.append(
    (
        "markdown_unsafe",
        lambda markdown_text: cmarkgfm.github_flavored_markdown_to_html(markdown_text, options=CMARKGFM_UNSAFE_OPTIONS),
        "Markdown (unsafe)",
    )
)

class Page(ContentManageable):
    title = models.CharField(max_length=500)
    keywords = models.CharField(max_length=1000, blank=True, help_text="HTTP meta-keywords")
    description = models.TextField(blank=True, help_text="HTTP meta-description")
    path = models.CharField(max_length=500, validators=[is_valid_page_path], unique=True, db_index=True)
    content = MarkupField(markup_choices=RENDERERS, default_markup_type=DEFAULT_MARKUP_TYPE)
    is_published = models.BooleanField(default=True, db_index=True)
    content_type = models.CharField(max_length=150, default='text/html')
    template_name = models.CharField(
        max_length=100,
        blank=True,
        help_text="Example: 'pages/about.html'. If this isn't provided, the system will use 'pages/default.html'."
    )

    objects = PageQuerySet.as_manager()

    class Meta:
        ordering = ['title', 'path']

    def clean(self):
        # Strip leading and trailing slashes off self.path.
        self.path = self.path.strip('/')

    def get_title(self):
        if self.title:
            return self.title
        else:
            return '** No Title **'

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return f"/{self.path}/"

@receiver(post_save, sender=Page)
def purge_fastly_cache(sender, instance, **kwargs):
    """
    Purge fastly.com cache if in production and the page is published.
    Requires settings.FASTLY_API_KEY being set
    """
    purge_url(f'/{instance.path}')

def page_image_path(instance, filename):
    return os.path.join(instance.page.path, filename)

class Image(models.Model):
    page = models.ForeignKey('pages.Page', on_delete=models.CASCADE)
    image = models.ImageField(upload_to=page_image_path, max_length=400)

    def __str__(self):
        return self.image.url

class DocumentFile(models.Model):
    page = models.ForeignKey('pages.Page', on_delete=models.CASCADE)
    document = models.FileField(upload_to='files/', max_length=500)

    def __str__(self):
        return self.document.url

<commit_msg>Use a custom version of the Github MD formatter (#2183)<commit_after>"""
Simple "flat pages".

These get used for the static (non-automated) large chunks of content. Notice
that pages don't have any actual notion of where they live; instead, they're
positioned into the URL structure using the nav app.
"""

import os
import re

from copy import deepcopy

from django.conf import settings
from django.core import validators
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver

from markupfield.fields import MarkupField
from markupfield.markup import DEFAULT_MARKUP_TYPES

import cmarkgfm
from cmarkgfm.cmark import Options as cmarkgfmOptions

from cms.models import ContentManageable
from fastly.utils import purge_url

from .managers import PageQuerySet

DEFAULT_MARKUP_TYPE = getattr(settings, 'DEFAULT_MARKUP_TYPE', 'restructuredtext')

PAGE_PATH_RE = re.compile(r"""
    ^
    /?                      # We can optionally start with a /
    ([a-z0-9-\.]+)            # Then at least one path segment...
    (/[a-z0-9-\.]+)*        # And then possibly more "/whatever" segments
    /?                      # Possibly ending with a slash
    $
    """,
    re.X
)

is_valid_page_path = validators.RegexValidator(
    regex=PAGE_PATH_RE,
    message=(
        'Please enter a valid URL segment, e.g. "foo" or "foo/bar". '
        'Only lowercase letters, numbers, hyphens and periods are allowed.'
    ),
)

RENDERERS = deepcopy(DEFAULT_MARKUP_TYPES)
for i, renderer in enumerate(RENDERERS):
    if renderer[0] == 'markdown':
        markdown_index = i

RENDERERS[markdown_index] = (
    'markdown',
    cmarkgfm.github_flavored_markdown_to_html,
    'Markdown'
)

# Add our own Github style Markdown parser, which doesn't apply the default
# tagfilter used by Github (we can be more liberal, since we know our page
# editors).

def unsafe_markdown_to_html(text, options=0):

    """Render the given GitHub-flavored Makrdown to HTML.

    This function is similar to cmarkgfm.github_flavored_markdown_to_html(),
    except that it allows raw HTML to get rendered, which is useful when
    using jQuery UI script extensions on pages.

    """
    # Set options for cmarkgfm for "unsafe" renderer, see
    # https://github.com/theacodes/cmarkgfm#advanced-usage
    options = options | (
        cmarkgfmOptions.CMARK_OPT_UNSAFE |
        cmarkgfmOptions.CMARK_OPT_GITHUB_PRE_LANG
    )
    return cmarkgfm.markdown_to_html_with_extensions(
        text, options=options,
        extensions=[
            'table', 'autolink', 'strikethrough', 'tasklist'
        ])

RENDERERS.append(
    (
        "markdown_unsafe",
        unsafe_markdown_to_html,
        "Markdown (unsafe)",
    )
)

class Page(ContentManageable):
    title = models.CharField(max_length=500)
    keywords = models.CharField(max_length=1000, blank=True, help_text="HTTP meta-keywords")
    description = models.TextField(blank=True, help_text="HTTP meta-description")
    path = models.CharField(max_length=500, validators=[is_valid_page_path], unique=True, db_index=True)
    content = MarkupField(markup_choices=RENDERERS, default_markup_type=DEFAULT_MARKUP_TYPE)
    is_published = models.BooleanField(default=True, db_index=True)
    content_type = models.CharField(max_length=150, default='text/html')
    template_name = models.CharField(
        max_length=100,
        blank=True,
        help_text="Example: 'pages/about.html'. If this isn't provided, the system will use 'pages/default.html'."
    )

    objects = PageQuerySet.as_manager()

    class Meta:
        ordering = ['title', 'path']

    def clean(self):
        # Strip leading and trailing slashes off self.path.
        self.path = self.path.strip('/')

    def get_title(self):
        if self.title:
            return self.title
        else:
            return '** No Title **'

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return f"/{self.path}/"

@receiver(post_save, sender=Page)
def purge_fastly_cache(sender, instance, **kwargs):
    """
    Purge fastly.com cache if in production and the page is published.
    Requires settings.FASTLY_API_KEY being set
    """
    purge_url(f'/{instance.path}')

def page_image_path(instance, filename):
    return os.path.join(instance.page.path, filename)

class Image(models.Model):
    page = models.ForeignKey('pages.Page', on_delete=models.CASCADE)
    image = models.ImageField(upload_to=page_image_path, max_length=400)

    def __str__(self):
        return self.image.url

class DocumentFile(models.Model):
    page = models.ForeignKey('pages.Page', on_delete=models.CASCADE)
    document = models.FileField(upload_to='files/', max_length=500)

    def __str__(self):
        return self.document.url

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Script to push a cleaned data file to the MUTI API
Requires a python environment with `requests` available in it.
There's currently nothing preventing things from being added again if you run
 the script more than once over the same meeting ids
Meeting ids only have to be unique within the file currently being pushed.
"""

import csv
import json
import requests

from argparse import ArgumentParser

BASE_API_URL = 'http://localhost:3000/api/v1'

CACHE = {
    'government-offices': {},
    'meetings': {},
    'organisations': {},
    'people': {},
}

def post_to_api_and_return_id(endpoint, data):
    url = '{0}/{1}'.format(BASE_API_URL, endpoint)
    resp = requests.post(url, json=data, headers={'Content-Type': 'application/vnd.api+json'})
    resp_data = resp.json()
    if not resp.ok:
        print("POST %s failed:\n%d %s" % (url, resp.status_code, resp.reason))
        print json.dumps(resp_data, sort_keys=True,
                         indent=4, separators=(',', ': '))
        resp.raise_for_status()
    if 'data' not in resp_data or not resp_data['data']:
        return None
    return resp_data["data"]["id"]

def lookup_from_api(endpoint, name):
    url = '{0}/{1}?filter[name]={2}'.format(BASE_API_URL, endpoint, name)
    resp = requests.get(url)
    data = resp.json()['data']
    if not data:
        return None
    return data[0]["id"]

def create_minister_link(meeting_id, minister_id, department_id):
    type_ = "influence-government-office-people"
    req_data = {
        "data": {
            "type": type_,
            "relationships": {
                "meeting": {
                    "data": {
                        "type": "meetings",
                        "id": meeting_id
                    }
                },
                "government-office": {
                    "data": {
                        "type": "government-offices",
                        "id": department_id
                     }
                },
                "person": {
                     "data": {
                        "type": "people",
                        "id": minister_id
                     }
                }
            }
        }
    }
    return post_to_api_and_return_id(type_, req_data)

def create_rep_link(meeting_id, rep_id, organisation_id):
    type_ = "influence-organisation-people"
    req_data = {
        "data": {
            "type": type_,
            "relationships": {
                "meeting": {
                    "data": {
                        "type": "meetings",
                        "id": meeting_id
                    }
                },
                "organisation": {
                    "data": {
                        "type": "organisations",
                        "id": organisation_id
                     }
                },
                "person": {
                     "data": {
                        "type": "people",
                        "id": rep_id
                     }
                }
            }
        }
    }
    return post_to_api_and_return_id(type_, req_data)

def get_or_create_meeting_id(meeting_ref, date_, reason):
    if meeting_ref in CACHE['meetings']:
        return CACHE['meetings'][meeting_ref]
    req_data = {
        "data": {
            "type": "meetings",
            "attributes": {
                "purpose": reason,
            }
        }
    }
    date_parts = date_.split('-')
    if len(date_parts) > 0:
        req_data['data']['attributes']['year'] = date_parts[0]
    if len(date_parts) > 1:
        req_data['data']['attributes']['month'] = date_parts[1]
    if len(date_parts) > 2:
        req_data['data']['attributes']['day'] = date_parts[2]
    CACHE['meetings'][meeting_ref] = post_to_api_and_return_id('meetings', req_data)
    return CACHE['meetings'][meeting_ref]

def get_or_create_department_id(department):
    return get_or_create_entity_id('government-offices', department)

def get_or_create_person_id(name):
    return get_or_create_entity_id('people', name)

def get_or_create_organisation_id(name):
    return get_or_create_entity_id('organisations', name)

def get_or_create_entity_id(type_, name):
    if name in CACHE[type_]:
        return CACHE[type_][name]
    entity_id = lookup_from_api(type_, name)
    if entity_id:
        CACHE[type_][name] = entity_id
        return CACHE[type_][name]
    req_data = {
        "data": {
            "type": type_,
            "attributes": {
                "name": name,
            }
        }
    }
    CACHE[type_][name] = post_to_api_and_return_id(type_, req_data)
    return CACHE[type_][name]

def main():
    global BASE_API_URL
    parser = ArgumentParser()
    parser.add_argument("file", help="Push data from FILE", metavar="FILE")
    parser.add_argument("--from-id", dest="from_id", default=0, metavar="FROM_ID", help="Ignore rows with meeting id < FROM_ID")
    parser.add_argument("--to-id", dest="to_id", default=1000000, metavar="TO_ID", help="Ignore rows with meeting id > TO_ID")
    parser.add_argument("--api-base", dest="base_api_url", default=BASE_API_URL, metavar="BASE_API_URL", help="Push data to api at BASE_API_URL")
    args = parser.parse_args()
    BASE_API_URL = args.base_api_url
    print args.file, int(args.from_id), int(args.to_id)

    count = 0
    with open(args.file, 'rU') as fh:
        reader = csv.reader(fh)
        skip = True
        for row in reader:
            if skip:
                skip = False
                continue
            try:
                meeting_ref = int(row[0])
            except ValueError:
                print("Couldn't parse meeting id; ignoring row:")
                print(row)
                continue
            if int(meeting_ref) < args.from_id or int(meeting_ref) > args.to_id:
                continue
            minister = row[1]
            department = row[3]
            date_ = row[4]
            org = row[6]
            rep = row[7]
            reason = row[8]
            meeting_id = get_or_create_meeting_id(meeting_ref, date_, reason)
            department_id = get_or_create_department_id(department)
            minister_id = get_or_create_person_id(minister)
            organisation_id = get_or_create_organisation_id(org)
            rep_id = get_or_create_person_id(rep or 'Representative from {0}'.format(org))
            create_minister_link(meeting_id, minister_id, department_id)
            create_rep_link(meeting_id, rep_id, organisation_id)
            count += 1
            if count % 100 == 0:
                print("%d rows processed" % count)

if __name__ == '__main__':
    main()
<commit_msg>push_data.py: handle API POST errors even better<commit_after>#!/usr/bin/env python
"""
Script to push a cleaned data file to the MUTI API
Requires a python environment with `requests` available in it.
There's currently nothing preventing things from being added again if you run
 the script more than once over the same meeting ids
Meeting ids only have to be unique within the file currently being pushed.
"""

import csv
import json
import requests
import simplejson

from argparse import ArgumentParser

BASE_API_URL = 'http://localhost:3000/api/v1'

CACHE = {
    'government-offices': {},
    'meetings': {},
    'organisations': {},
    'people': {},
}

def pprint_json(json_data):
    print(json.dumps(json_data, sort_keys=True,
                     indent = 4, separators=(',', ': ')))

def post_to_api_and_return_id(endpoint, data):
    url = '{0}/{1}'.format(BASE_API_URL, endpoint)
    resp = requests.post(url, json=data, headers={'Content-Type': 'application/vnd.api+json'})

    if not resp.ok:
        print("POST %s failed:\n%d %s" % (url, resp.status_code, resp.reason))
        print("Request was:")
        pprint_json(data)

    try:
        resp_data = resp.json()
    except simplejson.scanner.JSONDecodeError:
        resp_data = None

    if not resp.ok:
        print("Response was:")
        if resp_data is None:
            print(resp.text)
        else:
            pprint_json(resp_data)

        resp.raise_for_status()

    if 'data' not in resp_data or not resp_data['data']:
        return None
    return resp_data["data"]["id"]

def lookup_from_api(endpoint, name):
    url = '{0}/{1}?filter[name]={2}'.format(BASE_API_URL, endpoint, name)
    resp = requests.get(url)
    data = resp.json()['data']
    if not data:
        return None
    return data[0]["id"]

def create_minister_link(meeting_id, minister_id, department_id):
    type_ = "influence-government-office-people"
    req_data = {
        "data": {
            "type": type_,
            "relationships": {
                "meeting": {
                    "data": {
                        "type": "meetings",
                        "id": meeting_id
                    }
                },
                "government-office": {
                    "data": {
                        "type": "government-offices",
                        "id": department_id
                     }
                },
                "person": {
                     "data": {
                        "type": "people",
                        "id": minister_id
                     }
                }
            }
        }
    }
    return post_to_api_and_return_id(type_, req_data)

def create_rep_link(meeting_id, rep_id, organisation_id):
    type_ = "influence-organisation-people"
    req_data = {
        "data": {
            "type": type_,
            "relationships": {
                "meeting": {
                    "data": {
                        "type": "meetings",
                        "id": meeting_id
                    }
                },
                "organisation": {
                    "data": {
                        "type": "organisations",
                        "id": organisation_id
                     }
                },
                "person": {
                     "data": {
                        "type": "people",
                        "id": rep_id
                     }
                }
            }
        }
    }
    return post_to_api_and_return_id(type_, req_data)

def get_or_create_meeting_id(meeting_ref, date_, reason):
    if meeting_ref in CACHE['meetings']:
        return CACHE['meetings'][meeting_ref]
    req_data = {
        "data": {
            "type": "meetings",
            "attributes": {
                "purpose": reason,
            }
        }
    }
    date_parts = date_.split('-')
    if len(date_parts) > 0:
        req_data['data']['attributes']['year'] = date_parts[0]
    if len(date_parts) > 1:
        req_data['data']['attributes']['month'] = date_parts[1]
    if len(date_parts) > 2:
        req_data['data']['attributes']['day'] = date_parts[2]
    CACHE['meetings'][meeting_ref] = post_to_api_and_return_id('meetings', req_data)
    return CACHE['meetings'][meeting_ref]

def get_or_create_department_id(department):
    return get_or_create_entity_id('government-offices', department)

def get_or_create_person_id(name):
    return get_or_create_entity_id('people', name)

def get_or_create_organisation_id(name):
    return get_or_create_entity_id('organisations', name)

def get_or_create_entity_id(type_, name):
    if name in CACHE[type_]:
        return CACHE[type_][name]
    entity_id = lookup_from_api(type_, name)
    if entity_id:
        CACHE[type_][name] = entity_id
        return CACHE[type_][name]
    req_data = {
        "data": {
            "type": type_,
            "attributes": {
                "name": name,
            }
        }
    }
    CACHE[type_][name] = post_to_api_and_return_id(type_, req_data)
    return CACHE[type_][name]

def main():
    global BASE_API_URL
    parser = ArgumentParser()
    parser.add_argument("file", help="Push data from FILE", metavar="FILE")
    parser.add_argument("--from-id", dest="from_id", default=0, metavar="FROM_ID", help="Ignore rows with meeting id < FROM_ID")
    parser.add_argument("--to-id", dest="to_id", default=1000000, metavar="TO_ID", help="Ignore rows with meeting id > TO_ID")
    parser.add_argument("--api-base", dest="base_api_url", default=BASE_API_URL, metavar="BASE_API_URL", help="Push data to api at BASE_API_URL")
    args = parser.parse_args()
    BASE_API_URL = args.base_api_url
    print args.file, int(args.from_id), int(args.to_id)

    count = 0
    with open(args.file, 'rU') as fh:
        reader = csv.reader(fh)
        skip = True
        for row in reader:
            if skip:
                skip = False
                continue
            try:
                meeting_ref = int(row[0])
            except ValueError:
                print("Couldn't parse meeting id; ignoring row:")
                print(row)
                continue
            if int(meeting_ref) < args.from_id or int(meeting_ref) > args.to_id:
                continue
            minister = row[1]
            department = row[3]
            date_ = row[4]
            org = row[6]
            rep = row[7]
            reason = row[8]
            meeting_id = get_or_create_meeting_id(meeting_ref, date_, reason)
            department_id = get_or_create_department_id(department)
            minister_id = get_or_create_person_id(minister)
            organisation_id = get_or_create_organisation_id(org)
            rep_id = get_or_create_person_id(rep or 'Representative from {0}'.format(org))
            create_minister_link(meeting_id, minister_id, department_id)
            create_rep_link(meeting_id, rep_id, organisation_id)
            count += 1
            if count % 100 == 0:
                print("%d rows processed" % count)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import time
import logging
import numpy as np

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class FEI4SelfTriggerScan(ScanBase):
    def __init__(self, config_file, definition_file=None, bit_file=None, device=None, scan_identifier="scan_fei4_self_trigger", scan_data_path=None):
        super(FEI4SelfTriggerScan, self).__init__(config_file=config_file, definition_file=definition_file, bit_file=bit_file, device=device, scan_identifier=scan_identifier, scan_data_path=scan_data_path)

    def scan(self, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, scan_timeout=600, **kwargs):
        '''Scan loop

        Parameters
        ----------
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.
        '''
        # generate ROI mask for Enable mask
        pixel_reg = "Enable"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        enable_mask = np.logical_and(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, enable_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate ROI mask for Imon mask
        pixel_reg = "Imon"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        imon_mask = np.logical_or(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, imon_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # enable GateHitOr that enables FE self-trigger mode
        self.register.set_global_register_value("GateHitOr", 1)  # enable FE self-trigger mode
        self.register.set_global_register_value("Trig_Lat", 232)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", 0)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["GateHitOr", "Trig_Lat", "Trig_Count"]))
        commands.extend(self.register.get_commands("runmode"))
        # send commands
        self.register_utils.send_commands(commands)
        # preload command
        # lvl1_command = self.register.get_commands("zeros", length=24)[0]+self.register.get_commands("lv1")[0]#+BitVector.BitVector(size = 10000)
        # self.register_utils.set_command(lvl1_command)
        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()

            wait_for_first_data = True
            last_iteration = time.time()
            saw_no_data_at_time = last_iteration
            saw_data_at_time = last_iteration
            scan_start_time = last_iteration
            no_data_at_time = last_iteration
            time_from_last_iteration = 0
            scan_stop_time = scan_start_time + scan_timeout
            while not self.stop_thread_event.wait(self.readout.readout_interval):
    #                 if logger.isEnabledFor(logging.DEBUG):
    #                     lost_data = self.readout.get_lost_data_count()
    #                     if lost_data != 0:
    #                         logging.debug('Lost data count: %d', lost_data)
    #                         logging.debug('FIFO fill level: %4f', (float(fifo_size)/2**20)*100)
    #                         logging.debug('Collected triggers: %d', self.readout_utils.get_trigger_number())

                if scan_start_time is not None and time.time() > scan_stop_time:
                    logging.info('Reached maximum scan time. Stopping Scan...')
                    self.stop_thread_event.set()
                # TODO: read 8b10b decoder err cnt
    #                 if not self.readout_utils.read_rx_status():
    #                     logging.info('Lost data sync. Starting synchronization...')
    #                     self.readout_utils.set_ext_cmd_start(False)
    #                     if not self.readout_utils.reset_rx(1000):
    #                         logging.info('Failed. Stopping scan...')
    #                         self.stop_thread_event.set()
    #                     else:
    #                         logging.info('Done!')
    #                         self.readout_utils.set_ext_cmd_start(True)

                time_from_last_iteration = time.time() - last_iteration
                last_iteration = time.time()
                try:
                    raw_data_file.append((self.readout.data.popleft(),))
                    #logging.info('data words')
                except IndexError:  # no data
                    #logging.info('no data words')
                    no_data_at_time = last_iteration
                    if wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time

                    if no_data_at_time > (saw_data_at_time + 10):
                        scan_stop_time += time_from_last_iteration

                    continue

                saw_data_at_time = last_iteration

                if wait_for_first_data == True:
                    logging.info('Taking data...')
                    wait_for_first_data = False

            self.readout.stop()

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename + ".h5", analyzed_data_file=output_file) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.create_cluster_size_hist = True  # can be set to false to omit cluster hit creation, can save some time, std. setting is false
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.create_cluster_tot_hist = True
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms(scan_data_filename=scan.scan_data_filename)

if __name__ == "__main__":
    import configuration
    scan = FEI4SelfTriggerScan(config_file=configuration.config_file, bit_file=configuration.bit_file, scan_data_path=configuration.scan_data_path)
    scan.start(configure=True, use_thread=True, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, scan_timeout=10 * 60)
    scan.stop()
    scan.analyze()
<commit_msg>MAINT: remove kwargs<commit_after>import time
import logging
import numpy as np

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class FEI4SelfTriggerScan(ScanBase):
    def __init__(self, config_file, definition_file=None, bit_file=None, device=None, scan_identifier="scan_fei4_self_trigger", scan_data_path=None):
        super(FEI4SelfTriggerScan, self).__init__(config_file=config_file, definition_file=definition_file, bit_file=bit_file, device=device, scan_identifier=scan_identifier, scan_data_path=scan_data_path)

    def scan(self, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, scan_timeout=600):
        '''Scan loop

        Parameters
        ----------
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.
        '''
        # generate ROI mask for Enable mask
        pixel_reg = "Enable"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        enable_mask = np.logical_and(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, enable_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate ROI mask for Imon mask
        pixel_reg = "Imon"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        imon_mask = np.logical_or(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, imon_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # enable GateHitOr that enables FE self-trigger mode
        self.register.set_global_register_value("GateHitOr", 1)  # enable FE self-trigger mode
        self.register.set_global_register_value("Trig_Lat", 232)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", 0)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["GateHitOr", "Trig_Lat", "Trig_Count"]))
        commands.extend(self.register.get_commands("runmode"))
        # send commands
        self.register_utils.send_commands(commands)
        # preload command
        # lvl1_command = self.register.get_commands("zeros", length=24)[0]+self.register.get_commands("lv1")[0]#+BitVector.BitVector(size = 10000)
        # self.register_utils.set_command(lvl1_command)
        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()

            wait_for_first_data = True
            last_iteration = time.time()
            saw_no_data_at_time = last_iteration
            saw_data_at_time = last_iteration
            scan_start_time = last_iteration
            no_data_at_time = last_iteration
            time_from_last_iteration = 0
            scan_stop_time = scan_start_time + scan_timeout
            while not self.stop_thread_event.wait(self.readout.readout_interval):
    #                 if logger.isEnabledFor(logging.DEBUG):
    #                     lost_data = self.readout.get_lost_data_count()
    #                     if lost_data != 0:
    #                         logging.debug('Lost data count: %d', lost_data)
    #                         logging.debug('FIFO fill level: %4f', (float(fifo_size)/2**20)*100)
    #                         logging.debug('Collected triggers: %d', self.readout_utils.get_trigger_number())

                if scan_start_time is not None and time.time() > scan_stop_time:
                    logging.info('Reached maximum scan time. Stopping Scan...')
                    self.stop_thread_event.set()
                # TODO: read 8b10b decoder err cnt
    #                 if not self.readout_utils.read_rx_status():
    #                     logging.info('Lost data sync. Starting synchronization...')
    #                     self.readout_utils.set_ext_cmd_start(False)
    #                     if not self.readout_utils.reset_rx(1000):
    #                         logging.info('Failed. Stopping scan...')
    #                         self.stop_thread_event.set()
    #                     else:
    #                         logging.info('Done!')
    #                         self.readout_utils.set_ext_cmd_start(True)

                time_from_last_iteration = time.time() - last_iteration
                last_iteration = time.time()
                try:
                    raw_data_file.append((self.readout.data.popleft(),))
                    #logging.info('data words')
                except IndexError:  # no data
                    #logging.info('no data words')
                    no_data_at_time = last_iteration
                    if wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time

                    if no_data_at_time > (saw_data_at_time + 10):
                        scan_stop_time += time_from_last_iteration

                    continue

                saw_data_at_time = last_iteration

                if wait_for_first_data == True:
                    logging.info('Taking data...')
                    wait_for_first_data = False

            self.readout.stop()

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename + ".h5", analyzed_data_file=output_file) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.create_cluster_size_hist = True  # can be set to false to omit cluster hit creation, can save some time, std. setting is false
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.create_cluster_tot_hist = True
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms(scan_data_filename=scan.scan_data_filename)

if __name__ == "__main__":
    import configuration
    scan = FEI4SelfTriggerScan(config_file=configuration.config_file, bit_file=configuration.bit_file, scan_data_path=configuration.scan_data_path)
    scan.start(configure=True, use_thread=True, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, scan_timeout=10 * 60)
    scan.stop()
    scan.analyze()
<|endoftext|>


<commit_before>"""
--- Part Two ---

As the door slides open, you are presented with a second door that uses a slightly more inspired security mechanism. Clearly unimpressed by the last version (in what movie is the password decrypted in order?!), the Easter Bunny engineers have worked out a better solution.

Instead of simply filling in the password from left to right, the hash now also indicates the position within the password to fill. You still look for hashes that begin with five zeroes; however, now, the sixth character represents the position (0-7), and the seventh character is the character to put in that position.

A hash result of 000001f means that f is the second character in the password. Use only the first result for each position, and ignore invalid positions.

For example, if the Door ID is abc:

The first interesting hash is from abc3231929, which produces 0000015...; so, 5 goes in position 1: _5______.
In the previous method, 5017308 produced an interesting hash; however, it is ignored, because it specifies an invalid position (8).
The second interesting hash is at index 5357525, which produces 000004e...; so, e goes in position 4: _5__e___.
You almost choke on your popcorn as the final character falls into place, producing the password 05ace8e3.

Given the actual Door ID and this new method, what is the password? Be extra proud of your solution if it uses a cinematic "decrypting" animation.

Your puzzle input is still wtnhxymk.
"""
import argparse
import hashlib

def md5sum(decoded_string):
    md5 = hashlib.new("md5")
    md5.update(decoded_string.encode("utf-8"))
    return md5.hexdigest()

def calc_password(door_id):
    password = ["_"] * 8
    extra_char = 0
    print("".join(password), end = "\r")
    while "_" in password:
        door_id_md5 = md5sum((door_id + str(extra_char)))
        try:
            if door_id_md5.startswith("00000") and password[int(door_id_md5[5])] == "_":
                password[int(door_id_md5[5])] = door_id_md5[6]
        except Exception:
            pass
        print("".join(password), end="\r")
        extra_char += 1
    return "".join(password)

parser = argparse.ArgumentParser(description='Advent of code.')
parser.add_argument('inputfile', type=argparse.FileType('r'), help='Path to input file')
args = parser.parse_args()
lines = args.inputfile.read().rstrip("\n").split("\n")
door_id = lines[0]

print(calc_password(door_id))
<commit_msg>fixed day5-2 so it only prints code when the code changes<commit_after>"""
--- Part Two ---

As the door slides open, you are presented with a second door that uses a slightly more inspired security mechanism. Clearly unimpressed by the last version (in what movie is the password decrypted in order?!), the Easter Bunny engineers have worked out a better solution.

Instead of simply filling in the password from left to right, the hash now also indicates the position within the password to fill. You still look for hashes that begin with five zeroes; however, now, the sixth character represents the position (0-7), and the seventh character is the character to put in that position.

A hash result of 000001f means that f is the second character in the password. Use only the first result for each position, and ignore invalid positions.

For example, if the Door ID is abc:

The first interesting hash is from abc3231929, which produces 0000015...; so, 5 goes in position 1: _5______.
In the previous method, 5017308 produced an interesting hash; however, it is ignored, because it specifies an invalid position (8).
The second interesting hash is at index 5357525, which produces 000004e...; so, e goes in position 4: _5__e___.
You almost choke on your popcorn as the final character falls into place, producing the password 05ace8e3.

Given the actual Door ID and this new method, what is the password? Be extra proud of your solution if it uses a cinematic "decrypting" animation.

Your puzzle input is still wtnhxymk.
"""
import argparse
import hashlib

def md5sum(decoded_string):
    md5 = hashlib.new("md5")
    md5.update(decoded_string.encode("utf-8"))
    return md5.hexdigest()

def calc_password(door_id):
    password = ["_"] * 8
    extra_char = 0
    print("".join(password), end = "\r")
    while "_" in password:
        door_id_md5 = md5sum((door_id + str(extra_char)))
        try:
            if door_id_md5.startswith("00000") and password[int(door_id_md5[5])] == "_":
                password[int(door_id_md5[5])] = door_id_md5[6]
                print("".join(password), end="\r")
        except Exception:
            pass
        extra_char += 1
    return "".join(password)

parser = argparse.ArgumentParser(description='Advent of code.')
parser.add_argument('inputfile', type=argparse.FileType('r'), help='Path to input file')
args = parser.parse_args()
lines = args.inputfile.read().rstrip("\n").split("\n")
door_id = lines[0]

print(calc_password(door_id))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import optparse
import platform
import time
import copy
import subprocess
import shutil
import tempfile

path0 = os.path.dirname(os.path.abspath(__file__))
path3 = os.path.abspath(os.path.join(path0, os.pardir, os.pardir, 'util'))
sys.path.append(path3)
from downloadBuild import mozPlatformDetails
import subprocesses as sps

def parseOptions(inputArgs):
    """Returns a 'buildOptions' object, which is intended to be immutable."""

    parser = optparse.OptionParser()
    # http://docs.python.org/library/optparse.html#optparse.OptionParser.disable_interspersed_args
    parser.disable_interspersed_args()

    parser.set_defaults(
        repoDir = sps.normExpUserPath(os.path.join('~', 'trees', 'mozilla-central')),
        objDir = None,
        mozconfig = None
    )

    parser.add_option('-R', '--repoDir', dest='repoDir',
                      help='Sets the source repository. Defaults to "%default".')
    parser.add_option('-o', '--objDir', dest='objDir',
                      help='The obj dir that will be created by the given mozconfig. May get clobbered.')
    parser.add_option('-c', '--mozconfig', dest='mozconfig',
                      help='A mozconfig file.')

    (options, args) = parser.parse_args(inputArgs)

    if len(args) > 0:
        parser.print_help()
        raise Exception("buildBrowser.py: extra arguments")

    # All are required for now
    if not (options.objDir and options.repoDir and options.mozconfig):
        print "buildBrowser requires you to specify a repoDir, objDir, and mozconfig"
        parser.print_help()
        raise Exception("buildBrowser.py: usage")

    options.objDir = os.path.expanduser(options.objDir)
    options.repoDir = os.path.expanduser(options.repoDir)
    options.mozconfig = os.path.expanduser(options.mozconfig)

    assert os.path.exists(options.repoDir)
    assert os.path.exists(options.mozconfig)

    return options

def tryCompiling(options):
    env = copy.deepcopy(os.environ)
    env['MOZCONFIG'] = options.mozconfig

    compileOutput = tempfile.NamedTemporaryFile(delete=False)
    compileOutputFn = compileOutput.name
    print "Compiling (details in " + compileOutputFn + ")..."
    rv = subprocess.call(['make', '-C', options.repoDir, '-f', 'client.mk'], env=env, stdout=compileOutput.file, stderr=subprocess.STDOUT)
    compileOutput.close()
    if rv != 0:
        print "Compilation failed"
        time.sleep(8)
        if os.path.exists(options.objDir):
            shutil.rmtree(options.objDir)
        os.remove(compileOutputFn)
        return False
    else:
        assert os.path.exists(options.objDir)
        os.remove(compileOutputFn)
        return True

# For autoBisect
def makeTestRev(options):
    srcDir = options.browserOptions.repoDir
    objDir = options.browserOptions.objDir

    def testRev(rev):
        print "Updating to " + rev + "..."
        subprocess.check_call(['hg', '-R', srcDir, 'update', '-r', rev])
        sps.destroyPyc(srcDir)

        if os.path.exists(objDir):
            print "Clobbering..."
            # We don't trust the clobberer while bisecting
            shutil.rmtree(objDir)

        if not tryCompiling(options.browserOptions):
            return (options.compilationFailedLabel, "compilation failed")

        print "Testing..."
        ans = options.testAndLabel(objDir, rev)
        time.sleep(8)
        return ans

    return testRev

if __name__ == "__main__":
    print tryCompiling(parseOptions(sys.argv[1:]))
<commit_msg>Remove unused import.<commit_after>#!/usr/bin/env python

import os
import sys
import optparse
import platform
import time
import copy
import subprocess
import shutil
import tempfile

path0 = os.path.dirname(os.path.abspath(__file__))
path3 = os.path.abspath(os.path.join(path0, os.pardir, os.pardir, 'util'))
sys.path.append(path3)
import subprocesses as sps

def parseOptions(inputArgs):
    """Returns a 'buildOptions' object, which is intended to be immutable."""

    parser = optparse.OptionParser()
    # http://docs.python.org/library/optparse.html#optparse.OptionParser.disable_interspersed_args
    parser.disable_interspersed_args()

    parser.set_defaults(
        repoDir = sps.normExpUserPath(os.path.join('~', 'trees', 'mozilla-central')),
        objDir = None,
        mozconfig = None
    )

    parser.add_option('-R', '--repoDir', dest='repoDir',
                      help='Sets the source repository. Defaults to "%default".')
    parser.add_option('-o', '--objDir', dest='objDir',
                      help='The obj dir that will be created by the given mozconfig. May get clobbered.')
    parser.add_option('-c', '--mozconfig', dest='mozconfig',
                      help='A mozconfig file.')

    (options, args) = parser.parse_args(inputArgs)

    if len(args) > 0:
        parser.print_help()
        raise Exception("buildBrowser.py: extra arguments")

    # All are required for now
    if not (options.objDir and options.repoDir and options.mozconfig):
        print "buildBrowser requires you to specify a repoDir, objDir, and mozconfig"
        parser.print_help()
        raise Exception("buildBrowser.py: usage")

    options.objDir = os.path.expanduser(options.objDir)
    options.repoDir = os.path.expanduser(options.repoDir)
    options.mozconfig = os.path.expanduser(options.mozconfig)

    assert os.path.exists(options.repoDir)
    assert os.path.exists(options.mozconfig)

    return options

def tryCompiling(options):
    env = copy.deepcopy(os.environ)
    env['MOZCONFIG'] = options.mozconfig

    compileOutput = tempfile.NamedTemporaryFile(delete=False)
    compileOutputFn = compileOutput.name
    print "Compiling (details in " + compileOutputFn + ")..."
    rv = subprocess.call(['make', '-C', options.repoDir, '-f', 'client.mk'], env=env, stdout=compileOutput.file, stderr=subprocess.STDOUT)
    compileOutput.close()
    if rv != 0:
        print "Compilation failed"
        time.sleep(8)
        if os.path.exists(options.objDir):
            shutil.rmtree(options.objDir)
        os.remove(compileOutputFn)
        return False
    else:
        assert os.path.exists(options.objDir)
        os.remove(compileOutputFn)
        return True

# For autoBisect
def makeTestRev(options):
    srcDir = options.browserOptions.repoDir
    objDir = options.browserOptions.objDir

    def testRev(rev):
        print "Updating to " + rev + "..."
        subprocess.check_call(['hg', '-R', srcDir, 'update', '-r', rev])
        sps.destroyPyc(srcDir)

        if os.path.exists(objDir):
            print "Clobbering..."
            # We don't trust the clobberer while bisecting
            shutil.rmtree(objDir)

        if not tryCompiling(options.browserOptions):
            return (options.compilationFailedLabel, "compilation failed")

        print "Testing..."
        ans = options.testAndLabel(objDir, rev)
        time.sleep(8)
        return ans

    return testRev

if __name__ == "__main__":
    print tryCompiling(parseOptions(sys.argv[1:]))
<|endoftext|>


<commit_before>import itertools
import struct
import time
import datetime
import logging
from collections import deque
import math

import numpy as np
from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format = "%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class FEI4SelfTriggerScan(ScanBase):
    def __init__(self, config_file, definition_file = None, bit_file = None, device = None, scan_identifier = "scan_ext_trigger", scan_data_path = None):
        super(FEI4SelfTriggerScan, self).__init__(config_file = config_file, definition_file = definition_file, bit_file = bit_file, device = device, scan_identifier = scan_identifier, scan_data_path = scan_data_path)

    def scan(self, **kwargs):
        col_span = [1,80]
        row_span = [1,336]
        # generate ROI mask for Enable mask
        pixel_reg = "Enable"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc = False, name = pixel_reg))
        # generate ROI mask for Imon mask
        pixel_reg = "Imon"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc = False, name = pixel_reg))
        # enable GateHitOr that enables FE self-trigger mode
        self.register.set_global_register_value("GateHitOr", 1)
        commands.extend(self.register.get_commands("wrregister", name = "GateHitOr"))
        commands.extend(self.register.get_commands("runmode"))
        # send commands
        self.register_utils.send_commands(commands)
        # preload command
        #lvl1_command = self.register.get_commands("zeros", length=24)[0]+self.register.get_commands("lv1")[0]#+BitVector.BitVector(size = 10000)
        #self.register_utils.set_command(lvl1_command)
        with open_raw_data_file(filename = self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()

            wait_for_first_data = True

            timeout_no_data = 60 # secs
            scan_timeout = 1200
            last_iteration = time.time()
            saw_no_data_at_time = last_iteration
            saw_data_at_time = last_iteration
            scan_start_time = last_iteration
            no_data_at_time = last_iteration
            time_from_last_iteration = 0
            scan_stop_time = scan_start_time + scan_timeout
            while not self.stop_thread_event.wait(self.readout.readout_interval):

    #                 if logger.isEnabledFor(logging.DEBUG):
    #                     lost_data = self.readout.get_lost_data_count()
    #                     if lost_data != 0:
    #                         logging.debug('Lost data count: %d', lost_data)
    #                         logging.debug('FIFO fill level: %4f', (float(fifo_size)/2**20)*100)
    #                         logging.debug('Collected triggers: %d', self.readout_utils.get_trigger_number())

                if scan_start_time is not None and time.time() > scan_stop_time:
                    logging.info('Reached maximum scan time. Stopping Scan...')
                    self.stop_thread_event.set()
                # TODO: read 8b10b decoder err cnt
    #                 if not self.readout_utils.read_rx_status():
    #                     logging.info('Lost data sync. Starting synchronization...')
    #                     self.readout_utils.set_ext_cmd_start(False)
    #                     if not self.readout_utils.reset_rx(1000):
    #                         logging.info('Failed. Stopping scan...')
    #                         self.stop_thread_event.set()
    #                     else:
    #                         logging.info('Done!')
    #                         self.readout_utils.set_ext_cmd_start(True)

                time_from_last_iteration = time.time() - last_iteration
                last_iteration = time.time()
                try:
                    raw_data_file.append((self.readout.data.popleft()))
                    logging.info('data words')
                except IndexError:
                    logging.info('no data words')
                    no_data_at_time = last_iteration
                    if wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time

                    if no_data_at_time > (saw_data_at_time + 10):
                        scan_stop_time += time_from_last_iteration

                    continue

                saw_data_at_time = last_iteration

                if wait_for_first_data == True:
                    logging.info('Taking data...')
                    wait_for_first_data = False

            self.stop_thread_event.set()

            self.readout.stop()

if __name__ == "__main__":
    import configuration
    scan = FEI4SelfTriggerScan(config_file = configuration.config_file, bit_file  = configuration.bit_file, scan_data_path = configuration.scan_data_path)
    scan.start(configure=True, use_thread = False)
    scan.stop()
<commit_msg>BUG: fix tuple<commit_after>import itertools
import struct
import time
import datetime
import logging
from collections import deque
import math

import numpy as np
from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format = "%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class FEI4SelfTriggerScan(ScanBase):
    def __init__(self, config_file, definition_file = None, bit_file = None, device = None, scan_identifier = "scan_ext_trigger", scan_data_path = None):
        super(FEI4SelfTriggerScan, self).__init__(config_file = config_file, definition_file = definition_file, bit_file = bit_file, device = device, scan_identifier = scan_identifier, scan_data_path = scan_data_path)

    def scan(self, **kwargs):
        col_span = [1,80]
        row_span = [1,336]
        # generate ROI mask for Enable mask
        pixel_reg = "Enable"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc = False, name = pixel_reg))
        # generate ROI mask for Imon mask
        pixel_reg = "Imon"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc = False, name = pixel_reg))
        # enable GateHitOr that enables FE self-trigger mode
        self.register.set_global_register_value("GateHitOr", 1)
        commands.extend(self.register.get_commands("wrregister", name = "GateHitOr"))
        commands.extend(self.register.get_commands("runmode"))
        # send commands
        self.register_utils.send_commands(commands)
        # preload command
        #lvl1_command = self.register.get_commands("zeros", length=24)[0]+self.register.get_commands("lv1")[0]#+BitVector.BitVector(size = 10000)
        #self.register_utils.set_command(lvl1_command)
        with open_raw_data_file(filename = self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()

            wait_for_first_data = True

            timeout_no_data = 60 # secs
            scan_timeout = 1200
            last_iteration = time.time()
            saw_no_data_at_time = last_iteration
            saw_data_at_time = last_iteration
            scan_start_time = last_iteration
            no_data_at_time = last_iteration
            time_from_last_iteration = 0
            scan_stop_time = scan_start_time + scan_timeout
            while not self.stop_thread_event.wait(self.readout.readout_interval):

    #                 if logger.isEnabledFor(logging.DEBUG):
    #                     lost_data = self.readout.get_lost_data_count()
    #                     if lost_data != 0:
    #                         logging.debug('Lost data count: %d', lost_data)
    #                         logging.debug('FIFO fill level: %4f', (float(fifo_size)/2**20)*100)
    #                         logging.debug('Collected triggers: %d', self.readout_utils.get_trigger_number())

                if scan_start_time is not None and time.time() > scan_stop_time:
                    logging.info('Reached maximum scan time. Stopping Scan...')
                    self.stop_thread_event.set()
                # TODO: read 8b10b decoder err cnt
    #                 if not self.readout_utils.read_rx_status():
    #                     logging.info('Lost data sync. Starting synchronization...')
    #                     self.readout_utils.set_ext_cmd_start(False)
    #                     if not self.readout_utils.reset_rx(1000):
    #                         logging.info('Failed. Stopping scan...')
    #                         self.stop_thread_event.set()
    #                     else:
    #                         logging.info('Done!')
    #                         self.readout_utils.set_ext_cmd_start(True)

                time_from_last_iteration = time.time() - last_iteration
                last_iteration = time.time()
                try:
                    raw_data_file.append((self.readout.data.popleft(),))
                    logging.info('data words')
                except IndexError:
                    logging.info('no data words')
                    no_data_at_time = last_iteration
                    if wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time

                    if no_data_at_time > (saw_data_at_time + 10):
                        scan_stop_time += time_from_last_iteration

                    continue

                saw_data_at_time = last_iteration

                if wait_for_first_data == True:
                    logging.info('Taking data...')
                    wait_for_first_data = False

            self.stop_thread_event.set()

            self.readout.stop()

if __name__ == "__main__":
    import configuration
    scan = FEI4SelfTriggerScan(config_file = configuration.config_file, bit_file  = configuration.bit_file, scan_data_path = configuration.scan_data_path)
    scan.start(configure=True, use_thread = False)
    scan.stop()
<|endoftext|>


<commit_before>from ..exceptions import InvalidTld, UnsupportedTld
import idna
from ..models import (
    Domain,
    TopLevelDomain,
    TopLevelDomainProvider,
    RegisteredDomain,
    NameserverHost,
    IpAddress,
    Nameserver,
)

def get_domain_registry(fqdn):
    """
    Fetch the registry for a given domain.
    In the future there may be more than one upstream provider for a particular
    domain. If it is one registered in our system it should already be
    connected to one, but if not we should default to one particular registry.

    :fqdn: str domain
    :returns: DomainProvider object

    """
    parsed_domain = parse_domain(fqdn)
    tld_provider = None
    try:
        top_level_domain = TopLevelDomain.objects.get(
            zone=parsed_domain["zone"]
        )
        domain_obj, _ = Domain.objects.get_or_create(
            name=parsed_domain["domain"],
        )
        registered_domain_set = RegisteredDomain.objects.filter(
            domain=domain_obj,
            tld=top_level_domain,
            active=True
        )
        if registered_domain_set.count() > 0:
            registered_domain = registered_domain_set.first()
            tld_provider = registered_domain.tld_provider
        else:
            tld_provider = TopLevelDomainProvider.objects.get(
                zone=top_level_domain
            )
        return tld_provider.provider
    except TopLevelDomainProvider.DoesNotExist:
        raise UnsupportedTld(parsed_domain["zone"])
    except Exception as e:
        raise e

def parse_domain(fqdn):
    """
    Parse a domain into name and tld components.

    Note: resolution of a domain will depend on the TLDs that are currently
    provided by our TopLevelDomain table.

    :fqdn: a fully qualified domain name
    :returns: dict containing name and tld

    """
    fqdn = idna.encode(fqdn, uts46=True).decode('ascii')
    tlds = TopLevelDomain.objects.all()
    probable_tld = None
    length = 0
    domain_name = None
    for tld in tlds:
        zone = "." + tld.zone
        endindex = - (len(zone))
        if zone == fqdn[endindex:] and len(zone) > length:
            probable_tld = tld
            length = len(zone)
            # Get the actual domain name. Make sure it doesn't have
            # any subdomain prefixed
            domain_name = fqdn[:endindex].split(".")[-1]
    if domain_name is None or len(domain_name) == 0:
        raise InvalidTld(fqdn)
    return {"domain": domain_name, "zone": probable_tld.zone}

def synchronise_domain_nameserver(registered_domain, nameserver):
    """
    Add a nameserver to registered domain.

    :registered_domain: registered domain object
    :nameserver: str ns host

    """
    ns, _ = Nameserver.objects.get_or_create(idn_host=nameserver)
    registered_domain.ns.add(ns)

def synchronise_domain(info_data, domain_id):
    """
    Synchronise data in info domain response with upstream registry.

    :info_data: dict containing info data from registry
    :registered_domain: int primary key of domain
    """
    filtered_domains = RegisteredDomain.objects.filter(pk=domain_id)
    filtered_domains.update(
        authcode=info_data.get("authcode", None),
        roid=info_data.get("roid", None),
        status=info_data.get("status", None)
    )
    registered_domain = filtered_domains.first()
    if registered_domain and 'ns' in info_data:
        if isinstance(info_data['ns'], list):
            for ns in info_data['ns']:
                synchronise_domain_nameserver(registered_domain, ns)
        else:
            synchronise_domain_nameserver(registered_domain,
                                          info_data['ns'])

def synchronise_host(info_data, host_id):
    """
    Synchronise data in info host response from upstream registry.

    :info_data: dict containing info data from the registry
    :host_id: int primary key of host

    """
    nshost = NameserverHost.objects.filter(pk=host_id)
    nshost.update(
        status=info_data["status"],
        roid=info_data["roid"]
    )
    nshost_obj = nshost.get(pk=host_id)
    for addr in info_data["addr"]:
        address_type = addr["addr_type"]
        ip = addr["ip"]
        if not IpAddress.objects.filter(ip=ip, nameserver_host=nshost_obj).exists():
            IpAddress.objects.get_or_create(
                ip=ip,
                nameserver_host=nshost_obj,
                address_type=address_type,
                project_id=nshost_obj.project_id
            )
<commit_msg>Remove existing nameservers before adding set from registry<commit_after>from ..exceptions import InvalidTld, UnsupportedTld
import idna
from ..models import (
    Domain,
    TopLevelDomain,
    TopLevelDomainProvider,
    RegisteredDomain,
    NameserverHost,
    IpAddress,
    Nameserver,
)

def get_domain_registry(fqdn):
    """
    Fetch the registry for a given domain.
    In the future there may be more than one upstream provider for a particular
    domain. If it is one registered in our system it should already be
    connected to one, but if not we should default to one particular registry.

    :fqdn: str domain
    :returns: DomainProvider object

    """
    parsed_domain = parse_domain(fqdn)
    tld_provider = None
    try:
        top_level_domain = TopLevelDomain.objects.get(
            zone=parsed_domain["zone"]
        )
        domain_obj, _ = Domain.objects.get_or_create(
            name=parsed_domain["domain"],
        )
        registered_domain_set = RegisteredDomain.objects.filter(
            domain=domain_obj,
            tld=top_level_domain,
            active=True
        )
        if registered_domain_set.count() > 0:
            registered_domain = registered_domain_set.first()
            tld_provider = registered_domain.tld_provider
        else:
            tld_provider = TopLevelDomainProvider.objects.get(
                zone=top_level_domain
            )
        return tld_provider.provider
    except TopLevelDomainProvider.DoesNotExist:
        raise UnsupportedTld(parsed_domain["zone"])
    except Exception as e:
        raise e

def parse_domain(fqdn):
    """
    Parse a domain into name and tld components.

    Note: resolution of a domain will depend on the TLDs that are currently
    provided by our TopLevelDomain table.

    :fqdn: a fully qualified domain name
    :returns: dict containing name and tld

    """
    fqdn = idna.encode(fqdn, uts46=True).decode('ascii')
    tlds = TopLevelDomain.objects.all()
    probable_tld = None
    length = 0
    domain_name = None
    for tld in tlds:
        zone = "." + tld.zone
        endindex = - (len(zone))
        if zone == fqdn[endindex:] and len(zone) > length:
            probable_tld = tld
            length = len(zone)
            # Get the actual domain name. Make sure it doesn't have
            # any subdomain prefixed
            domain_name = fqdn[:endindex].split(".")[-1]
    if domain_name is None or len(domain_name) == 0:
        raise InvalidTld(fqdn)
    return {"domain": domain_name, "zone": probable_tld.zone}

def synchronise_domain_nameserver(registered_domain, nameserver):
    """
    Add a nameserver to registered domain.

    :registered_domain: registered domain object
    :nameserver: str ns host

    """
    ns, _ = Nameserver.objects.get_or_create(idn_host=nameserver)
    registered_domain.ns.add(ns)

def synchronise_domain(info_data, domain_id):
    """
    Synchronise data in info domain response with upstream registry.

    :info_data: dict containing info data from registry
    :registered_domain: int primary key of domain
    """
    filtered_domains = RegisteredDomain.objects.filter(pk=domain_id)
    filtered_domains.update(
        authcode=info_data.get("authcode", None),
        roid=info_data.get("roid", None),
        status=info_data.get("status", None)
    )
    registered_domain = filtered_domains.first()
    if registered_domain and 'ns' in info_data:
        current_nameservers = registered_domain.ns.all()
        # Remove existing nameservers
        for current_nameserver in current_nameservers:
            registered_domain.ns.remove(current_nameserver)
        if isinstance(info_data['ns'], list):
            for ns in info_data['ns']:
                synchronise_domain_nameserver(registered_domain, ns)
        else:
            synchronise_domain_nameserver(registered_domain,
                                          info_data['ns'])

def synchronise_host(info_data, host_id):
    """
    Synchronise data in info host response from upstream registry.

    :info_data: dict containing info data from the registry
    :host_id: int primary key of host

    """
    nshost = NameserverHost.objects.filter(pk=host_id)
    nshost.update(
        status=info_data["status"],
        roid=info_data["roid"]
    )
    nshost_obj = nshost.get(pk=host_id)
    for addr in info_data["addr"]:
        address_type = addr["addr_type"]
        ip = addr["ip"]
        if not IpAddress.objects.filter(ip=ip, nameserver_host=nshost_obj).exists():
            IpAddress.objects.get_or_create(
                ip=ip,
                nameserver_host=nshost_obj,
                address_type=address_type,
                project_id=nshost_obj.project_id
            )
<|endoftext|>


<commit_before>import time
import logging
import math
import numpy as np

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

local_configuration = {
    "col_span": [2, 77],
    "row_span": [2, 335],
    "timeout_no_data": 10,
    "scan_timeout": 1 * 60,
    "trig_latency": 239,
    "trig_count": 4
}

class FEI4SelfTriggerScan(ScanBase):
    scan_identifier = "fei4_self_trigger_scan"

    def scan(self, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, scan_timeout=1 * 60, trig_latency=239, trig_count=4, **kwargs):
        '''Scan loop

        Parameters
        ----------
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.
        trig_latency : int
            FE global register Trig_Lat.
        trig_count : int
            FE global register Trig_Count.
        '''

        self.configure_fe(col_span, row_span, trig_latency, trig_count)

        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()
            self.set_self_trigger(True)
            wait_for_first_data = True
            show_trigger_message_at = 10 ** (int(math.floor(math.log10(scan_timeout) - math.log10(3) / math.log10(10))))
            time_current_iteration = time.time()
            saw_no_data_at_time = time_current_iteration
            saw_data_at_time = time_current_iteration
            scan_start_time = time_current_iteration
            no_data_at_time = time_current_iteration
            time_from_last_iteration = 0
            scan_stop_time = scan_start_time + scan_timeout
            while not self.stop_thread_event.wait(self.readout.readout_interval):
                time_last_iteration = time_current_iteration
                time_current_iteration = time.time()
                time_from_last_iteration = time_current_iteration - time_last_iteration
                if ((time_current_iteration - scan_start_time) % show_trigger_message_at < (time_last_iteration - scan_start_time) % show_trigger_message_at):
                    logging.info('Scan runtime: %d seconds', time_current_iteration - scan_start_time)
                    if not any(self.readout.get_rx_sync_status()):
                        self.stop_thread_event.set()
                        logging.error('No RX sync. Stopping Scan...')
                    if any(self.readout.get_rx_8b10b_error_count()):
                        self.stop_thread_event.set()
                        logging.error('RX 8b10b error(s) detected. Stopping Scan...')
                    if any(self.readout.get_rx_fifo_discard_count()):
                        self.stop_thread_event.set()
                        logging.error('RX FIFO discard error(s) detected. Stopping Scan...')
                if scan_timeout is not None and time_current_iteration > scan_stop_time:
                    logging.info('Reached maximum scan time. Stopping Scan...')
                    self.stop_thread_event.set()
                try:
                    raw_data_file.append((self.readout.data.popleft(),))
                except IndexError:  # no data
                    no_data_at_time = time_current_iteration
                    if timeout_no_data is not None and wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time

                    if no_data_at_time > (saw_data_at_time + 10):
                        scan_stop_time += time_from_last_iteration
                else:
                    saw_data_at_time = time_current_iteration

                    if wait_for_first_data == True:
                        logging.info('Taking data...')
                        wait_for_first_data = False

            self.set_self_trigger(False)
            self.readout.stop()

            raw_data_file.append(self.readout.data)

    def configure_fe(self, col_span, row_span, trig_latency, trig_count):
        # generate ROI mask for Enable mask
        pixel_reg = "Enable"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        enable_mask = np.logical_and(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, enable_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate ROI mask for Imon mask
        pixel_reg = "Imon"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        imon_mask = np.logical_or(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, imon_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # enable GateHitOr that enables FE self-trigger mode
        self.register.set_global_register_value("Trig_Lat", trig_latency)  # set trigger latency, this latency sets the hits at the first relative BCID bins
        self.register.set_global_register_value("Trig_Count", trig_count)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["Trig_Lat", "Trig_Count"]))
        # send commands
        self.register_utils.send_commands(commands)

    def set_self_trigger(self, enable=True):
        logging.info('%s FEI4 self-trigger' % ('Enable' if enable == True else "Disable"))
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_global_register_value("GateHitOr", 1 if enable else 0)  # enable FE self-trigger mode
        commands.extend(self.register.get_commands("wrregister", name=["GateHitOr"]))
        if enable:
            commands.extend(self.register.get_commands("runmode"))
        self.register_utils.send_commands(commands)

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename + ".h5", analyzed_data_file=output_file) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.create_cluster_size_hist = True  # can be set to false to omit cluster hit creation, can save some time, standard setting is false
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.create_cluster_tot_hist = True
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.clusterizer.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms(scan_data_filename=scan.scan_data_filename)

if __name__ == "__main__":
    import configuration
    scan = FEI4SelfTriggerScan(**configuration.default_configuration)
    scan.start(use_thread=True, **local_configuration)
    scan.stop()
    scan.analyze()
<commit_msg>BUG: now the IMON/HITBUS mask is set to make the scan work without manually changing configuration files<commit_after>import time
import logging
import math
import numpy as np

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

local_configuration = {
    "col_span": [2, 77],
    "row_span": [2, 335],
    "timeout_no_data": 10,
    "scan_timeout": 1 * 60,
    "trig_latency": 239,
    "trig_count": 4
}

class FEI4SelfTriggerScan(ScanBase):
    scan_identifier = "fei4_self_trigger_scan"

    def scan(self, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, scan_timeout=1 * 60, trig_latency=239, trig_count=4, **kwargs):
        '''Scan loop

        Parameters
        ----------
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.
        trig_latency : int
            FE global register Trig_Lat.
        trig_count : int
            FE global register Trig_Count.
        '''

        self.configure_fe(col_span, row_span, trig_latency, trig_count)

        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()
            self.set_self_trigger(True)
            wait_for_first_data = True
            show_trigger_message_at = 10 ** (int(math.floor(math.log10(scan_timeout) - math.log10(3) / math.log10(10))))
            time_current_iteration = time.time()
            saw_no_data_at_time = time_current_iteration
            saw_data_at_time = time_current_iteration
            scan_start_time = time_current_iteration
            no_data_at_time = time_current_iteration
            time_from_last_iteration = 0
            scan_stop_time = scan_start_time + scan_timeout
            while not self.stop_thread_event.wait(self.readout.readout_interval):
                time_last_iteration = time_current_iteration
                time_current_iteration = time.time()
                time_from_last_iteration = time_current_iteration - time_last_iteration
                if ((time_current_iteration - scan_start_time) % show_trigger_message_at < (time_last_iteration - scan_start_time) % show_trigger_message_at):
                    logging.info('Scan runtime: %d seconds', time_current_iteration - scan_start_time)
                    if not any(self.readout.get_rx_sync_status()):
                        self.stop_thread_event.set()
                        logging.error('No RX sync. Stopping Scan...')
                    if any(self.readout.get_rx_8b10b_error_count()):
                        self.stop_thread_event.set()
                        logging.error('RX 8b10b error(s) detected. Stopping Scan...')
                    if any(self.readout.get_rx_fifo_discard_count()):
                        self.stop_thread_event.set()
                        logging.error('RX FIFO discard error(s) detected. Stopping Scan...')
                if scan_timeout is not None and time_current_iteration > scan_stop_time:
                    logging.info('Reached maximum scan time. Stopping Scan...')
                    self.stop_thread_event.set()
                try:
                    raw_data_file.append((self.readout.data.popleft(),))
                except IndexError:  # no data
                    no_data_at_time = time_current_iteration
                    if timeout_no_data is not None and wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time

                    if no_data_at_time > (saw_data_at_time + 10):
                        scan_stop_time += time_from_last_iteration
                else:
                    saw_data_at_time = time_current_iteration

                    if wait_for_first_data == True:
                        logging.info('Taking data...')
                        wait_for_first_data = False

            self.set_self_trigger(False)
            self.readout.stop()

            raw_data_file.append(self.readout.data)

    def configure_fe(self, col_span, row_span, trig_latency, trig_count):
        # generate ROI mask for Enable mask
        pixel_reg = "Enable"
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        enable_mask = np.logical_and(mask, self.register.get_pixel_register_value(pixel_reg))
        self.register.set_pixel_register_value(pixel_reg, enable_mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate ROI mask for Imon mask
        pixel_reg = "Imon"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # enable GateHitOr that enables FE self-trigger mode
        self.register.set_global_register_value("Trig_Lat", trig_latency)  # set trigger latency, this latency sets the hits at the first relative BCID bins
        self.register.set_global_register_value("Trig_Count", trig_count)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["Trig_Lat", "Trig_Count"]))
        # send commands
        self.register_utils.send_commands(commands)

    def set_self_trigger(self, enable=True):
        logging.info('%s FEI4 self-trigger' % ('Enable' if enable == True else "Disable"))
        commands = []
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_global_register_value("GateHitOr", 1 if enable else 0)  # enable FE self-trigger mode
        commands.extend(self.register.get_commands("wrregister", name=["GateHitOr"]))
        if enable:
            commands.extend(self.register.get_commands("runmode"))
        self.register_utils.send_commands(commands)

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename + ".h5", analyzed_data_file=output_file) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.create_cluster_size_hist = True  # can be set to false to omit cluster hit creation, can save some time, standard setting is false
            analyze_raw_data.create_source_scan_hist = True
            analyze_raw_data.create_cluster_tot_hist = True
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.clusterizer.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms(scan_data_filename=scan.scan_data_filename)

if __name__ == "__main__":
    import configuration
    scan = FEI4SelfTriggerScan(**configuration.default_configuration)
    scan.start(use_thread=True, **local_configuration)
    scan.stop()
    scan.analyze()
<|endoftext|>


<commit_before>from logging import FileHandler
import logging

from flask import Flask, render_template, request, jsonify
import psycopg2

from dbManager import Queries, dbpass
from model import genre_list

# This file is the first that is called when someone goes to a page on our website.
# It looks at the url that they entered and decides what computation to do and what
#   template to show them.

app = Flask(__name__)

app.config['APP_NAME'] = "Don't Care Radio"
app.config.from_pyfile('../conf/flask.conf.py')
file_handler = FileHandler('/var/log/flask/info.log')
file_handler.setLevel(logging.INFO)
app.logger.addHandler(file_handler)
page_size = 10

try:
    db_conn = psycopg2.connect("dbname=radiodb user=radiodb host=localhost password=%s" % dbpass)
except psycopg2.DatabaseError:
    app.logger.error("could not connect to db!")

@app.route('/')
def render_landing():
    return render_template("player.html")

@app.route('/get-stations/')
def get_stations():
    db_cur = db_conn.cursor()
    genre_names = genre_list[int(request.args.get('genre', ''))]
    app.logger.info("%s, %s", genre_names, request.args.get('page', ''))
    data = {
        'genre_names': genre_names,
        'page_number': page_size * int(request.args.get('page', '')),
        'page_size':   page_size
    }
    db_cur.execute(Queries.get_urls_by_genre, data)
    results = db_cur.fetchall()
    app.logger.info(str(results))
    db_cur.close()
    return jsonify(stations=results)

@app.route('/get-genre-count/')
def get_genre_count():
    return jsonify(genreCount=len(genre_list))

if __name__ == '__main__':
    app.run()
<commit_msg>Changed path for flask template dir.<commit_after>from logging import FileHandler
import logging

from flask import Flask, render_template, request, jsonify
import psycopg2

from dbManager import Queries, dbpass
from model import genre_list

# This file is the first that is called when someone goes to a page on our website.
# It looks at the url that they entered and decides what computation to do and what
#   template to show them.

app = Flask(__name__, template_folder="../templates")

app.config['APP_NAME'] = "Don't Care Radio"
app.config.from_pyfile('../conf/flask.conf.py')
file_handler = FileHandler('/var/log/flask/info.log')
file_handler.setLevel(logging.INFO)
app.logger.addHandler(file_handler)
page_size = 10

try:
    db_conn = psycopg2.connect("dbname=radiodb user=radiodb host=localhost password=%s" % dbpass)
except psycopg2.DatabaseError:
    app.logger.error("could not connect to db!")

@app.route('/')
def render_landing():
    return render_template("player.html")

@app.route('/get-stations/')
def get_stations():
    db_cur = db_conn.cursor()
    genre_names = genre_list[int(request.args.get('genre', ''))]
    app.logger.info("%s, %s", genre_names, request.args.get('page', ''))
    data = {
        'genre_names': genre_names,
        'page_number': page_size * int(request.args.get('page', '')),
        'page_size':   page_size
    }
    db_cur.execute(Queries.get_urls_by_genre, data)
    results = db_cur.fetchall()
    app.logger.info(str(results))
    db_cur.close()
    return jsonify(stations=results)

@app.route('/get-genre-count/')
def get_genre_count():
    return jsonify(genreCount=len(genre_list))

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>import json
import re

import requests

import dbt.exceptions
import dbt.semver

PYPI_VERSION_URL = 'https://pypi.org/pypi/dbt/json'

def get_latest_version():
    resp = requests.get(PYPI_VERSION_URL)
    try:
        data = resp.json()
        version_string = data['info']['version']
    except (json.JSONDecodeError, KeyError):
        return None

    return dbt.semver.VersionSpecifier.from_version_string(version_string)

def get_installed_version():
    return dbt.semver.VersionSpecifier.from_version_string(__version__)

def get_version_information():
    installed = get_installed_version()
    latest = get_latest_version()

    installed_s = installed.to_version_string(skip_matcher=True)
    if latest is None:
        latest_s = 'unknown'
    else:
        latest_s = latest.to_version_string(skip_matcher=True)

    version_msg = ("installed version: {}\n"
                   "   latest version: {}\n\n".format(installed_s, latest_s))

    if latest is None:
        return ("{}The latest version of dbt could not be determined!\n"
                "Make sure that the following URL is accessible:\n{}"
                .format(version_msg, PYPI_VERSION_URL))

    if installed == latest:
        return "{}Up to date!".format(version_msg)

    elif installed > latest:
        return ("{}Your version of dbt is ahead of the latest "
                "release!".format(version_msg))

    else:
        return ("{}Your version of dbt is out of date! "
                "You can find instructions for upgrading here:\n"
                "https://docs.getdbt.com/docs/installation"
                .format(version_msg))

__version__ = '0.12.1'
installed = get_installed_version()
<commit_msg>more pr feedback<commit_after>import json
import re

import requests

import dbt.exceptions
import dbt.semver

PYPI_VERSION_URL = 'https://pypi.org/pypi/dbt/json'

def get_latest_version():
    try:
        resp = requests.get(PYPI_VERSION_URL)
        data = resp.json()
        version_string = data['info']['version']
    except (json.JSONDecodeError, KeyError, requests.RequestException):
        return None

    return dbt.semver.VersionSpecifier.from_version_string(version_string)

def get_installed_version():
    return dbt.semver.VersionSpecifier.from_version_string(__version__)

def get_version_information():
    installed = get_installed_version()
    latest = get_latest_version()

    installed_s = installed.to_version_string(skip_matcher=True)
    if latest is None:
        latest_s = 'unknown'
    else:
        latest_s = latest.to_version_string(skip_matcher=True)

    version_msg = ("installed version: {}\n"
                   "   latest version: {}\n\n".format(installed_s, latest_s))

    if latest is None:
        return ("{}The latest version of dbt could not be determined!\n"
                "Make sure that the following URL is accessible:\n{}"
                .format(version_msg, PYPI_VERSION_URL))

    if installed == latest:
        return "{}Up to date!".format(version_msg)

    elif installed > latest:
        return ("{}Your version of dbt is ahead of the latest "
                "release!".format(version_msg))

    else:
        return ("{}Your version of dbt is out of date! "
                "You can find instructions for upgrading here:\n"
                "https://docs.getdbt.com/docs/installation"
                .format(version_msg))

__version__ = '0.12.1'
installed = get_installed_version()
<|endoftext|>


<commit_before>'''
Tests of output_tables.py module
'''

import pytest
import os
import pandas as pd
from ogusa import utils, output_tables

# Load in test results and parameters
CUR_PATH = os.path.abspath(os.path.dirname(__file__))
base_ss = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'SS_vars_baseline.pkl'))
base_tpi = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'TPI_vars_baseline.pkl'))
base_params = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'model_params_baseline.pkl'))
reform_ss = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'SS_vars_reform.pkl'))
reform_tpi = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'TPI_vars_reform.pkl'))
reform_params = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'model_params_reform.pkl'))

def test_macro_table():
    df = output_tables.macro_table(base_tpi, base_params, reform_tpi,
                                   reform_params)
    assert isinstance(df, pd.DataFrame)

def test_macro_table_SS():
    df = output_tables.macro_table_SS(base_ss, reform_ss)
    assert isinstance(df, pd.DataFrame)
<commit_msg>test of ineq table<commit_after>'''
Tests of output_tables.py module
'''

import pytest
import os
import pandas as pd
from ogusa import utils, output_tables

# Load in test results and parameters
CUR_PATH = os.path.abspath(os.path.dirname(__file__))
base_ss = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'SS_vars_baseline.pkl'))
base_tpi = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'TPI_vars_baseline.pkl'))
base_params = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'model_params_baseline.pkl'))
reform_ss = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'SS_vars_reform.pkl'))
reform_tpi = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'TPI_vars_reform.pkl'))
reform_params = utils.safe_read_pickle(
    os.path.join(CUR_PATH, 'test_io_data', 'model_params_reform.pkl'))

def test_macro_table():
    df = output_tables.macro_table(base_tpi, base_params, reform_tpi,
                                   reform_params)
    assert isinstance(df, pd.DataFrame)

def test_macro_table_SS():
    df = output_tables.macro_table_SS(base_ss, reform_ss)
    assert isinstance(df, pd.DataFrame)

def test_ineq_table():
    df = output_tables.ineq_table(base_ss, base_params)
    assert isinstance(df, pd.DataFrame)
<|endoftext|>