<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This is the core script of the UnivaqInformaticaBot created by Giacomo Cerquone and Diego Mariani
"""

import telegram
import os.path

from telegram import Updater
from utils import utils

def start_command(bot, update):
    """Defining the `start` command"""

    welcome = "Ciao, sono il bot dell'Univaq (Università dell'Aquila).\n" \
              "Posso fornirti tutte le informazioni di cui hai bisogno sulla nostra università,"\
              "digita /help per vedere la lista di comandi."

    bot.sendMessage(update.message.chat_id, text=welcome)

def help_command(bot, update):
    """Defining the `help` command"""

    help_message = "La lista di comandi:\n\n" \
                   "/help - Stampa questo messaggio\n" \
                   "/news - Leggi le ultime 10 news\n" \
                   "/news num - Leggi le ultime <num> news\n" \
                   "/newson - Abilita le notifiche per ogni nuova news\n" \
                   "/newsoff - Disabilita le notifiche per ogni nuova news\n" \
                   "/prof - Stampa la lista dei professori\n" \
                   "/prof cognome - Info su un docente\n" \
                   "/segreteria - Info sulla segreteria studenti\n" \
                   "/mensa - Info sugli orari della mensa\n" \
                   "/adsu - Info sull'adsu" \
                   "\n\nQuesto bot è orgogliosamente open source, sviluppato da Giacomo Cerquone" \
                   " e Diego Mariani."

    bot.sendMessage(update.message.chat_id, text=help_message)

def news_command(bot, update, args):
    """Defining the `news` command"""

    if len(args) and int(args[0]) <= 10:
        ten_news = utils.pull_news(args[0])
    else:
        ten_news = utils.pull_news(10)

    ten_news_string = ""
    for i, news in enumerate(ten_news):
        truncated_descr = news['description'][:75] + '...' if len(news['description']) > 75 \
                          else news['description']
        ten_news_string += str(i+1) + "- " + news['title'] + "\n" + truncated_descr + "\n"

    bot.sendMessage(update.message.chat_id, text=ten_news_string)

def newson_command(bot, update):
    """Defining the command to enable notifications for news"""

    def notify_news(bat):
        """Defining method that will be repeated over and over"""
        unread_news = utils.check_news()

        if len(unread_news) > 0:
            data = utils.pull_news(10)
            utils.write_json(data, "json/news.json")
            new_news_string = ""
            for news in unread_news:
                new_news_string += news['title'] + "\n" + news['description'] + "\n\n"
            bat.sendMessage(update.message.chat_id, text=new_news_string)

    if(os.path.isfile("json/news.json") == False):
        print(os.path.isfile("json/news.json"))
        utils.write_json(utils.pull_news(10), "json/news.json")

    JOB_QUEUE.put(notify_news, 10, repeat=True)
    bot.sendMessage(update.message.chat_id, text='Notifiche abilitate')

def newsoff_command(bot, update):
    """Defining the command to disable notifications for news"""

    JOB_QUEUE.stop()
    bot.sendMessage(update.message.chat_id, text='Notifiche disabilitate')

def prof_command(bot, update, args):
    """Defining the `prof` command"""

    data = utils.read_json("json/professors.json")
    professors = ""

    if len(args):
        for prof in data:
            if args[0].lower() in prof['nome'].lower():
                professors += prof['nome'] + \
                              " - " + prof['telefono'] + \
                              " - " + prof['e-mail'] + \
                              " - " + prof['corsi'] + \
                              "\n\n"
    else:
        for prof in data:
            professors += prof['nome'] + \
                          " - " + prof['telefono'] + \
                          " - " + prof['e-mail'] + \
                          "\n"

    bot.sendMessage(update.message.chat_id, text=professors)

def student_office_command(bot, update):
    """Defining the `student_office` command"""

    data = utils.read_json("json/student_office.json")
    student_office_info = "Orari: " + data['orari'] + \
                          "\nIndirizzo: " + data['indirizzo'] + \
                          "\nTelefono: " + data['telefono'] + \
                          "\nE-mail: " + data['e-mail']

    bot.sendMessage(update.message.chat_id, text=student_office_info)

def canteen_command(bot, update):
    """Defining the `canteen` command"""

    data = utils.read_json("json/mensa.json")
    bot.sendMessage(update.message.chat_id, text="Orari: "+data['orari'])

def adsu_command(bot, update):
    """Defining the `canteen` command"""

    data = utils.read_json("json/adsu.json")
    bot.sendMessage(update.message.chat_id, text=data['info'])

# For testing only
def commands_keyboard(bot, update):
    """Enable a custom keyboard"""

    keyboard = [["/help", "/news", "/prof", "/mensa", "/cancel"]]
    markup = telegram.ReplyKeyboardMarkup(keyboard, resize_keyboard=True, one_time_keyboard=False)
    bot.sendMessage(update.message.chat_id, text="Enabled keyboard", reply_markup=markup)

def main():
    """Defining the main function"""

    global JOB_QUEUE

    config = utils.get_configuration()
    token = config.get('API-KEYS', 'TelegramBot')
    debug = config.getboolean('UTILS', 'Debug')
    logger = utils.get_logger(debug)

    updater = Updater(token)
    JOB_QUEUE = updater.job_queue
    dispatcher = updater.dispatcher

    dispatcher.addTelegramCommandHandler("start", start_command)
    dispatcher.addTelegramCommandHandler("help", help_command)
    dispatcher.addTelegramCommandHandler("news", news_command)
    dispatcher.addTelegramCommandHandler("newson", newson_command)
    dispatcher.addTelegramCommandHandler("newsoff", newsoff_command)
    dispatcher.addTelegramCommandHandler("prof", prof_command)
    dispatcher.addTelegramCommandHandler("segreteria", student_office_command)
    dispatcher.addTelegramCommandHandler("mensa", canteen_command)
    dispatcher.addTelegramCommandHandler("adsu", adsu_command)

    # For Testing only
    dispatcher.addTelegramCommandHandler("commands_keyboard", commands_keyboard)

    logger.info('Bot started')

    updater.start_polling()
    updater.idle()

if __name__ == '__main__':
    main()
<commit_msg>Linted code<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This is the core script of the UnivaqInformaticaBot created by Giacomo Cerquone and Diego Mariani
"""

import os.path
import telegram

from telegram import Updater
from utils import utils

def start_command(bot, update):
    """Defining the `start` command"""

    welcome = "Ciao, sono il bot dell'Univaq (Università dell'Aquila).\n" \
              "Posso fornirti tutte le informazioni di cui hai bisogno sulla nostra università,"\
              "digita /help per vedere la lista di comandi."

    bot.sendMessage(update.message.chat_id, text=welcome)

def help_command(bot, update):
    """Defining the `help` command"""

    help_message = "La lista di comandi:\n\n" \
                   "/help - Stampa questo messaggio\n" \
                   "/news - Leggi le ultime 10 news\n" \
                   "/news num - Leggi le ultime <num> news\n" \
                   "/newson - Abilita le notifiche per ogni nuova news (default)\n" \
                   "/newsoff - Disabilita le notifiche per ogni nuova news\n" \
                   "/prof - Stampa la lista dei professori\n" \
                   "/prof cognome - Info su un docente\n" \
                   "/segreteria - Info sulla segreteria studenti\n" \
                   "/mensa - Info sugli orari della mensa\n" \
                   "/adsu - Info sull'adsu" \
                   "\n\nQuesto bot è orgogliosamente open source, sviluppato da Giacomo Cerquone" \
                   " e Diego Mariani."

    bot.sendMessage(update.message.chat_id, text=help_message)

def news_command(bot, update, args):
    """Defining the `news` command"""

    if len(args) and int(args[0]) <= 10:
        ten_news = utils.pull_news(args[0])
    else:
        ten_news = utils.pull_news(10)

    ten_news_string = ""
    for i, news in enumerate(ten_news):
        truncated_descr = news['description'][:75] + '...' if len(news['description']) > 75 \
                          else news['description']
        ten_news_string += str(i+1) + "- " + news['title'] + "\n" + truncated_descr + "\n"

    bot.sendMessage(update.message.chat_id, text=ten_news_string)

def newson_command(bot, update):
    """Defining the command to enable notifications for news"""

    def notify_news(bat):
        """Defining method that will be repeated over and over"""
        unread_news = utils.check_news()

        if len(unread_news) > 0:
            data = utils.pull_news(10)
            utils.write_json(data, "json/news.json")
            new_news_string = ""
            for news in unread_news:
                new_news_string += news['title'] + "\n" + news['description'] + "\n\n"
            bat.sendMessage(update.message.chat_id, text=new_news_string)

    if not os.path.isfile("json/news.json"):
        print(os.path.isfile("json/news.json"))
        utils.write_json(utils.pull_news(10), "json/news.json")

    JOB_QUEUE.put(notify_news, 10, repeat=True)
    bot.sendMessage(update.message.chat_id, text='Notifiche abilitate')

def newsoff_command(bot, update):
    """Defining the command to disable notifications for news"""

    JOB_QUEUE.stop()
    bot.sendMessage(update.message.chat_id, text='Notifiche disabilitate')

def prof_command(bot, update, args):
    """Defining the `prof` command"""

    data = utils.read_json("json/professors.json")
    professors = ""

    if len(args):
        for prof in data:
            if args[0].lower() in prof['nome'].lower():
                professors += prof['nome'] + \
                              " - " + prof['telefono'] + \
                              " - " + prof['e-mail'] + \
                              " - " + prof['corsi'] + \
                              "\n\n"
    else:
        for prof in data:
            professors += prof['nome'] + \
                          " - " + prof['telefono'] + \
                          " - " + prof['e-mail'] + \
                          "\n"

    bot.sendMessage(update.message.chat_id, text=professors)

def student_office_command(bot, update):
    """Defining the `student_office` command"""

    data = utils.read_json("json/student_office.json")
    student_office_info = "Orari: " + data['orari'] + \
                          "\nIndirizzo: " + data['indirizzo'] + \
                          "\nTelefono: " + data['telefono'] + \
                          "\nE-mail: " + data['e-mail']

    bot.sendMessage(update.message.chat_id, text=student_office_info)

def canteen_command(bot, update):
    """Defining the `canteen` command"""

    data = utils.read_json("json/mensa.json")
    bot.sendMessage(update.message.chat_id, text="Orari: "+data['orari'])

def adsu_command(bot, update):
    """Defining the `canteen` command"""

    data = utils.read_json("json/adsu.json")
    bot.sendMessage(update.message.chat_id, text=data['info'])

# For testing only
def commands_keyboard(bot, update):
    """Enable a custom keyboard"""

    keyboard = [["/help", "/news", "/prof", "/mensa", "/cancel"]]
    markup = telegram.ReplyKeyboardMarkup(keyboard, resize_keyboard=True, one_time_keyboard=False)
    bot.sendMessage(update.message.chat_id, text="Enabled keyboard", reply_markup=markup)

def main():
    """Defining the main function"""

    global JOB_QUEUE

    config = utils.get_configuration()
    token = config.get('API-KEYS', 'TelegramBot')
    debug = config.getboolean('UTILS', 'Debug')
    logger = utils.get_logger(debug)

    updater = Updater(token)
    JOB_QUEUE = updater.job_queue
    dispatcher = updater.dispatcher

    dispatcher.addTelegramCommandHandler("start", start_command)
    dispatcher.addTelegramCommandHandler("help", help_command)
    dispatcher.addTelegramCommandHandler("news", news_command)
    dispatcher.addTelegramCommandHandler("newson", newson_command)
    dispatcher.addTelegramCommandHandler("newsoff", newsoff_command)
    dispatcher.addTelegramCommandHandler("prof", prof_command)
    dispatcher.addTelegramCommandHandler("segreteria", student_office_command)
    dispatcher.addTelegramCommandHandler("mensa", canteen_command)
    dispatcher.addTelegramCommandHandler("adsu", adsu_command)

    # For Testing only
    dispatcher.addTelegramCommandHandler("commands_keyboard", commands_keyboard)

    logger.info('Bot started')

    updater.start_polling()
    updater.idle()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
from lxml import etree
class device:
    def __init__(self,ipadress,name):
        self.ipadress=str(ipadress)
        self.name=str(name)
        self.status="off"
    def turn_on(self):
        self.status="on"
    def turn_off(self):
        self.status="off"

def getstatus(devices):
    ips=[]
    for instance in devices:
        instance.turn_off()
    test=os.popen("nmap -sP --unprivileged 192.168.2.0/24")
    for i in test.readlines():
        if i.split(' ')[0]=='Nmap' and i.split(' ')[1]=='scan' :
            ips.append(i.split('(')[1][:-2])
    for i in xrange(0,len(ips)):
        for j in xrange(0,len(devices)):
            if ips[i]== devices[j].ipadress:
                devices[j].turn_on()
    return devices

def writexmlrow(device,container):
    col=etree.SubElement(container,'div',{'class':'col-xs-6 col-sm-2 placeholder'})
    if (device.status=='on'):
        image1=etree.SubElement(col,'img',{'src':'./images/green.png','width':'200','height':'200','class':'img-responsive','align':'center'})
    else:
        image1=etree.SubElement(col,'img',{'src':'./images/gray.png','width':'200','height':'200','class':'img-responsive','align':'center'})
    label1=etree.SubElement(col,'h4',{'align':'center'})
    label1.text=device.name
    return

def writexmlpart(devices):
    container=etree.Element('div',{'class':'row placeholder'})
    for instance in devices:
        writexmlrow(instance,container)
    output=etree.tostring(container, pretty_print=True)
    with open("./parts/part1_1.html","r") as file:
        part1=file.read()
    with open("./parts/part1_2.html","r") as file:
        part2=file.read()
    with open("./parts/part1.html","w") as file:
        file.write(part1+output+part2)
    return
def writescanlog():
    localtime==time.localtime(time.time())
    with open("./log/scanlog.txt","a") as log:
        log.write(str(localtime[3])+':'+str(localtime[4])+'on the'+str(localtime[2])+'.'+str(localtime[1])+'.'+str(localtime[0])[-2:])
        log.write("Scanned Wifi for my Devices")
<commit_msg>Adjust Bootstrap Grid<commit_after>import os
from lxml import etree
class device:
    def __init__(self,ipadress,name):
        self.ipadress=str(ipadress)
        self.name=str(name)
        self.status="off"
    def turn_on(self):
        self.status="on"
    def turn_off(self):
        self.status="off"

def getstatus(devices):
    ips=[]
    for instance in devices:
        instance.turn_off()
    test=os.popen("nmap -sP --unprivileged 192.168.2.0/24")
    for i in test.readlines():
        if i.split(' ')[0]=='Nmap' and i.split(' ')[1]=='scan' :
            ips.append(i.split('(')[1][:-2])
    for i in xrange(0,len(ips)):
        for j in xrange(0,len(devices)):
            if ips[i]== devices[j].ipadress:
                devices[j].turn_on()
    return devices

def writexmlrow(device,container):
    col=etree.SubElement(container,'div',{'class':'col-xs-6 col-md-2 placeholder'})
    if (device.status=='on'):
        image1=etree.SubElement(col,'img',{'src':'./images/green.png','width':'200','height':'200','class':'img-responsive','align':'center'})
    else:
        image1=etree.SubElement(col,'img',{'src':'./images/gray.png','width':'200','height':'200','class':'img-responsive','align':'center'})
    label1=etree.SubElement(col,'h4',{'align':'center'})
    label1.text=device.name
    return

def writexmlpart(devices):
    container=etree.Element('div',{'class':'row placeholder'})
    for instance in devices:
        writexmlrow(instance,container)
    output=etree.tostring(container, pretty_print=True)
    with open("./parts/part1_1.html","r") as file:
        part1=file.read()
    with open("./parts/part1_2.html","r") as file:
        part2=file.read()
    with open("./parts/part1.html","w") as file:
        file.write(part1+output+part2)
    return
def writescanlog():
    localtime==time.localtime(time.time())
    with open("./log/scanlog.txt","a") as log:
        log.write(str(localtime[3])+':'+str(localtime[4])+'on the'+str(localtime[2])+'.'+str(localtime[1])+'.'+str(localtime[0])[-2:])
        log.write("Scanned Wifi for my Devices")
<|endoftext|>


<commit_before>import sys
import os
import shutil
import time
import random
from taichi.tools.video import make_video, interpolate_frames, mp4_to_gif, scale_video, crop_video, accelerate_video

def test_python():
  print("\nRunning python tests...\n")
  import taichi as ti
  import pytest
  if ti.is_release():
    return int(pytest.main([os.path.join(ti.package_root(), 'tests')]))
  else:
    return int(pytest.main([os.path.join(ti.get_repo_directory(), 'tests')]))

def test_cpp():
  import taichi as ti
  if not ti.core.with_cuda():
    print("Skipping legacy tests (no GPU support)")
    return 0
  # Cpp tests use the legacy non LLVM backend
  os.environ['TI_LLVM'] = '0'
  ti.reset()
  print("Running C++ tests...")
  task = ti.Task('test')
  return task.run(*sys.argv[2:])

def main(debug=False):
  lines = []
  print()
  lines.append(u' *******************************************')
  lines.append(u' **     Taichi Programming Language       **')
  lines.append(u' *******************************************')
  if debug:
    lines.append(u' *****************Debug Mode****************')
    os.environ['TI_DEBUG'] = '1'
  print(u'\n'.join(lines))
  print()
  import taichi as ti

  ti.tc_core.set_core_debug(debug)

  argc = len(sys.argv)
  if argc == 1 or sys.argv[1] == 'help':
    print(
        "    Usage: ti run [task name]        |-> Run a specific task\n"
        "           ti benchmark              |-> Run performance benchmark\n"
        "           ti test                   |-> Run all tests\n"
        "           ti test_python            |-> Run python tests\n"
        "           ti test_cpp               |-> Run cpp tests\n"
        "           ti format                 |-> Reformat modified source files\n"
        "           ti format_all             |-> Reformat all source files\n"
        "           ti build                  |-> Build C++ files\n"
        "           ti video                  |-> Make a video using *.png files in the current folder\n"
        "           ti video_scale            |-> Scale video resolution \n"
        "           ti video_crop             |-> Crop video\n"
        "           ti video_speed            |-> Speed up video\n"
        "           ti gif                    |-> Convert mp4 file to gif\n"
        "           ti doc                    |-> Build documentation\n"
        "           ti release                |-> Make source code release\n"
        "           ti debug [script.py]      |-> Debug script\n")
    exit(0)
  mode = sys.argv[1]

  t = time.time()
  if mode.endswith('.py'):
    import subprocess
    subprocess.call([sys.executable] + sys.argv[1:])
  elif mode == "run":
    if argc <= 2:
      print("Please specify [task name], e.g. test_math")
      exit(-1)
    name = sys.argv[2]
    task = ti.Task(name)
    task.run(*sys.argv[3:])
  elif mode == "debug":
    ti.core.set_core_trigger_gdb_when_crash(True)
    if argc <= 2:
      print("Please specify [file name], e.g. render.py")
      exit(-1)
    name = sys.argv[2]
    with open(name) as script:
      script = script.read()
    exec(script, {'__name__': '__main__'})
  elif mode == "test_python":
    return test_python()
  elif mode == "test_cpp":
    return test_cpp()
  elif mode == "test":
    if test_python() != 0:
      return -1
    return test_cpp()
  elif mode == "build":
    ti.core.build()
  elif mode == "format":
    ti.core.format()
  elif mode == "format_all":
    ti.core.format(all=True)
  elif mode == "statement":
    exec(sys.argv[2])
  elif mode == "update":
    ti.core.update(True)
    ti.core.build()
  elif mode == "asm":
    fn = sys.argv[2]
    os.system(r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn))
  elif mode == "interpolate":
    interpolate_frames('.')
  elif mode == "amal":
    cwd = os.getcwd()
    os.chdir(ti.get_repo_directory())
    with open('misc/amalgamate.py') as script:
      script = script.read()
    exec(script, {'__name__': '__main__'})
    os.chdir(cwd)
    shutil.copy(
        os.path.join(ti.get_repo_directory(), 'build/taichi.h'), './taichi.h')
  elif mode == "doc":
    os.system('cd {}/docs && sphinx-build -b html . build'.format(ti.get_repo_directory()))
  elif mode == "video":
    files = sorted(os.listdir('.'))
    files = list(filter(lambda x: x.endswith('.png'), files))
    if len(sys.argv) >= 3:
      frame_rate = int(sys.argv[2])
    else:
      frame_rate = 24
    if len(sys.argv) >= 4:
      trunc = int(sys.argv[3])
      files = files[:trunc]
    ti.info('Making video using {} png files...', len(files))
    ti.info("frame_rate={}", frame_rate)
    output_fn = 'video.mp4'
    make_video(files, output_path=output_fn, frame_rate=frame_rate)
    ti.info('Done! Output video file = {}', output_fn)
  elif mode == "video_scale":
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '-scaled.mp4'
    ratiow = float(sys.argv[3])
    if len(sys.argv) >= 5:
      ratioh = float(sys.argv[4])
    else:
      ratioh = ratiow
    scale_video(input_fn, output_fn, ratiow, ratioh)
  elif mode == "video_crop":
    if len(sys.argv) != 7:
      print('Usage: ti video_crop fn x_begin x_end y_begin y_end')
      exit(-1)
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '-cropped.mp4'
    x_begin = float(sys.argv[3])
    x_end = float(sys.argv[4])
    y_begin = float(sys.argv[5])
    y_end = float(sys.argv[6])
    crop_video(input_fn, output_fn, x_begin, x_end, y_begin, y_end)
  elif mode == "video_speed":
    if len(sys.argv) != 4:
      print('Usage: ti video_speed fn speed_up_factor')
      exit(-1)
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '-sped.mp4'
    speed = float(sys.argv[3])
    accelerate_video(input_fn, output_fn, speed)
  elif mode == "gif":
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '.gif'
    ti.info('Converting {} to {}'.format(input_fn, output_fn))
    framerate = 24
    mp4_to_gif(input_fn, output_fn, framerate)
  elif mode == "convert":
    # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed
    # TODO: Windows support
    for fn in sys.argv[2:]:
      print("Converting logging file: {}".format(fn))
      tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000))
      shutil.move(fn, tmp_fn)
      command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"'
      os.system('{} {} > {}'.format(command, tmp_fn, fn))
  elif mode == "release":
    from git import Git
    import zipfile
    import hashlib
    g = Git(ti.get_repo_directory())
    g.init()
    with zipfile.ZipFile('release.zip', 'w') as zip:
      files = g.ls_files().split('\n')
      os.chdir(ti.get_repo_directory())
      for f in files:
        if not os.path.isdir(f):
          zip.write(f)
    ver = ti.__version__
    md5 = hashlib.md5()
    with open('release.zip', "rb") as f:
      for chunk in iter(lambda: f.read(4096), b""):
        md5.update(chunk)
    md5 = md5.hexdigest()
    commit = ti.core.get_commit_hash()[:8]
    fn = f'taichi-src-v{ver[0]}-{ver[1]}-{ver[2]}-{commit}-{md5}.zip'
    import shutil
    shutil.move('release.zip', fn)
  else:
    name = sys.argv[1]
    print('Running task [{}]...'.format(name))
    task = ti.Task(name)
    task.run(*sys.argv[2:])
  print()
  print(">>> Running time: {:.2f}s".format(time.time() - t))

def main_debug():
  main(debug=True)

if __name__ == '__main__':
  exit(main())
<commit_msg>ti test_verbose<commit_after>import sys
import os
import shutil
import time
import random
from taichi.tools.video import make_video, interpolate_frames, mp4_to_gif, scale_video, crop_video, accelerate_video

def test_python(verbose=False):
  print("\nRunning python tests...\n")
  import taichi as ti
  import pytest
  if ti.is_release():
    test_dir = os.path.join(ti.package_root(), 'tests')
  else:
    test_dir = os.path.join(ti.get_repo_directory(), 'tests')

  args = [test_dir]
  if verbose:
    args += ['-s']

  return int(pytest.main(args))

def test_cpp():
  import taichi as ti
  if not ti.core.with_cuda():
    print("Skipping legacy tests (no GPU support)")
    return 0
  # Cpp tests use the legacy non LLVM backend
  os.environ['TI_LLVM'] = '0'
  ti.reset()
  print("Running C++ tests...")
  task = ti.Task('test')
  return task.run(*sys.argv[2:])

def main(debug=False):
  lines = []
  print()
  lines.append(u' *******************************************')
  lines.append(u' **     Taichi Programming Language       **')
  lines.append(u' *******************************************')
  if debug:
    lines.append(u' *****************Debug Mode****************')
    os.environ['TI_DEBUG'] = '1'
  print(u'\n'.join(lines))
  print()
  import taichi as ti

  ti.tc_core.set_core_debug(debug)

  argc = len(sys.argv)
  if argc == 1 or sys.argv[1] == 'help':
    print(
        "    Usage: ti run [task name]        |-> Run a specific task\n"
        "           ti benchmark              |-> Run performance benchmark\n"
        "           ti test                   |-> Run all tests\n"
        "           ti test_verbose           |-> Run all tests with verbose outputs\n"
        "           ti test_python            |-> Run python tests\n"
        "           ti test_cpp               |-> Run cpp tests\n"
        "           ti format                 |-> Reformat modified source files\n"
        "           ti format_all             |-> Reformat all source files\n"
        "           ti build                  |-> Build C++ files\n"
        "           ti video                  |-> Make a video using *.png files in the current folder\n"
        "           ti video_scale            |-> Scale video resolution \n"
        "           ti video_crop             |-> Crop video\n"
        "           ti video_speed            |-> Speed up video\n"
        "           ti gif                    |-> Convert mp4 file to gif\n"
        "           ti doc                    |-> Build documentation\n"
        "           ti release                |-> Make source code release\n"
        "           ti debug [script.py]      |-> Debug script\n")
    exit(0)
  mode = sys.argv[1]

  t = time.time()
  if mode.endswith('.py'):
    import subprocess
    subprocess.call([sys.executable] + sys.argv[1:])
  elif mode == "run":
    if argc <= 2:
      print("Please specify [task name], e.g. test_math")
      exit(-1)
    name = sys.argv[2]
    task = ti.Task(name)
    task.run(*sys.argv[3:])
  elif mode == "debug":
    ti.core.set_core_trigger_gdb_when_crash(True)
    if argc <= 2:
      print("Please specify [file name], e.g. render.py")
      exit(-1)
    name = sys.argv[2]
    with open(name) as script:
      script = script.read()
    exec(script, {'__name__': '__main__'})
  elif mode == "test_python":
    return test_python()
  elif mode == "test_cpp":
    return test_cpp()
  elif mode == "test":
    if test_python() != 0:
      return -1
    return test_cpp()
  elif mode == "test_verbose":
    if test_python(True) != 0:
      return -1
    return test_cpp()
  elif mode == "build":
    ti.core.build()
  elif mode == "format":
    ti.core.format()
  elif mode == "format_all":
    ti.core.format(all=True)
  elif mode == "statement":
    exec(sys.argv[2])
  elif mode == "update":
    ti.core.update(True)
    ti.core.build()
  elif mode == "asm":
    fn = sys.argv[2]
    os.system(r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn))
  elif mode == "interpolate":
    interpolate_frames('.')
  elif mode == "amal":
    cwd = os.getcwd()
    os.chdir(ti.get_repo_directory())
    with open('misc/amalgamate.py') as script:
      script = script.read()
    exec(script, {'__name__': '__main__'})
    os.chdir(cwd)
    shutil.copy(
        os.path.join(ti.get_repo_directory(), 'build/taichi.h'), './taichi.h')
  elif mode == "doc":
    os.system('cd {}/docs && sphinx-build -b html . build'.format(ti.get_repo_directory()))
  elif mode == "video":
    files = sorted(os.listdir('.'))
    files = list(filter(lambda x: x.endswith('.png'), files))
    if len(sys.argv) >= 3:
      frame_rate = int(sys.argv[2])
    else:
      frame_rate = 24
    if len(sys.argv) >= 4:
      trunc = int(sys.argv[3])
      files = files[:trunc]
    ti.info('Making video using {} png files...', len(files))
    ti.info("frame_rate={}", frame_rate)
    output_fn = 'video.mp4'
    make_video(files, output_path=output_fn, frame_rate=frame_rate)
    ti.info('Done! Output video file = {}', output_fn)
  elif mode == "video_scale":
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '-scaled.mp4'
    ratiow = float(sys.argv[3])
    if len(sys.argv) >= 5:
      ratioh = float(sys.argv[4])
    else:
      ratioh = ratiow
    scale_video(input_fn, output_fn, ratiow, ratioh)
  elif mode == "video_crop":
    if len(sys.argv) != 7:
      print('Usage: ti video_crop fn x_begin x_end y_begin y_end')
      exit(-1)
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '-cropped.mp4'
    x_begin = float(sys.argv[3])
    x_end = float(sys.argv[4])
    y_begin = float(sys.argv[5])
    y_end = float(sys.argv[6])
    crop_video(input_fn, output_fn, x_begin, x_end, y_begin, y_end)
  elif mode == "video_speed":
    if len(sys.argv) != 4:
      print('Usage: ti video_speed fn speed_up_factor')
      exit(-1)
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '-sped.mp4'
    speed = float(sys.argv[3])
    accelerate_video(input_fn, output_fn, speed)
  elif mode == "gif":
    input_fn = sys.argv[2]
    assert input_fn[-4:] == '.mp4'
    output_fn = input_fn[:-4] + '.gif'
    ti.info('Converting {} to {}'.format(input_fn, output_fn))
    framerate = 24
    mp4_to_gif(input_fn, output_fn, framerate)
  elif mode == "convert":
    # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed
    # TODO: Windows support
    for fn in sys.argv[2:]:
      print("Converting logging file: {}".format(fn))
      tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000))
      shutil.move(fn, tmp_fn)
      command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"'
      os.system('{} {} > {}'.format(command, tmp_fn, fn))
  elif mode == "release":
    from git import Git
    import zipfile
    import hashlib
    g = Git(ti.get_repo_directory())
    g.init()
    with zipfile.ZipFile('release.zip', 'w') as zip:
      files = g.ls_files().split('\n')
      os.chdir(ti.get_repo_directory())
      for f in files:
        if not os.path.isdir(f):
          zip.write(f)
    ver = ti.__version__
    md5 = hashlib.md5()
    with open('release.zip', "rb") as f:
      for chunk in iter(lambda: f.read(4096), b""):
        md5.update(chunk)
    md5 = md5.hexdigest()
    commit = ti.core.get_commit_hash()[:8]
    fn = f'taichi-src-v{ver[0]}-{ver[1]}-{ver[2]}-{commit}-{md5}.zip'
    import shutil
    shutil.move('release.zip', fn)
  else:
    name = sys.argv[1]
    print('Running task [{}]...'.format(name))
    task = ti.Task(name)
    task.run(*sys.argv[2:])
  print()
  print(">>> Running time: {:.2f}s".format(time.time() - t))

def main_debug():
  main(debug=True)

if __name__ == '__main__':
  exit(main())
<|endoftext|>


<commit_before>class Calculator:

    OPERATORS = {"plus": "+",
                 "minus": "-",
                 "multiplied by": "*",
                 "divided by": "/",
                 "What is ": "",
                 "?": ""}

    VALID_TOKENS = set(OPERATORS.values())

    @classmethod
    def calculate(cls, inp):
        inp = cls.tokenize(inp)
        if not cls.valid(inp):
            raise ValueError
        return eval(inp)

    @classmethod
    def valid(cls, inp):
        return (cls.valid_elements(inp) and
                not cls.consecutive_tokens(inp) and
                not cls.consecutive_digits(inp))

    @classmethod
    def consecutive_tokens(cls, inp):
        return any(i in cls.OPERATORS.values() and j in cls.OPERATORS.values()
                   for i, j in cls.slices_of_two(inp))

    @classmethod
    def consecutive_digits(cls, inp):
        return any(cls.digit(i) and cls.digit(j) for i, j in
                   cls.slices_of_two(inp))

    @classmethod
    def slices_of_two(cls, inp):
        return zip(inp.split(" "), inp.split(" ")[1:])

    @classmethod
    def valid_elements(cls, inp):
        return all(cls.valid_element(element) for element in inp.split(" "))

    @classmethod
    def valid_element(cls, element):
        return element in cls.VALID_TOKENS or cls.digit(element)

    @classmethod
    def tokenize(cls, inp):
        for operator, token in cls.OPERATORS.items():
            inp = inp.replace(operator, token)
        return inp

    @staticmethod
    def digit(element):
        return element.lstrip("-").isdigit()

def calculate(inp):
    return Calculator.calculate(inp)
<commit_msg>Wordy: class to instance methods<commit_after>class Calculator:

    OPERATORS = {"plus": "+",
                 "minus": "-",
                 "multiplied by": "*",
                 "divided by": "/",
                 "What is ": "",
                 "?": ""}

    VALID_TOKENS = set(OPERATORS.values())

    def __init__(self, inp):
        self.inp = inp
        self.tokenized = self.tokenize(inp)
        self.tokens = self.tokenized.split(" ")

    def calculate(self):
        if not self.valid():
            raise ValueError
        return eval(self.tokenized)

    def valid(self):
        return (self.valid_elements() and
                not self.consecutive_tokens() and
                not self.consecutive_digits())

    def consecutive_tokens(self):
        return any(i in self.OPERATORS.values() and
                   j in self.OPERATORS.values() for i, j
                   in self.slices_of_two())

    def consecutive_digits(self):
        return any(self.digit(i) and self.digit(j) for i, j in
                   self.slices_of_two())

    def slices_of_two(self):
        return zip(self.tokens, self.tokens[1:])

    def valid_elements(self):
        return all(self.valid_element(element) for element in self.tokens)

    @classmethod
    def valid_element(cls, element):
        return element in cls.VALID_TOKENS or cls.digit(element)

    @classmethod
    def tokenize(cls, inp):
        for operator, token in cls.OPERATORS.items():
            inp = inp.replace(operator, token)
        return inp

    @staticmethod
    def digit(element):
        return element.lstrip("-").isdigit()

def calculate(inp):
    return Calculator(inp).calculate()
<|endoftext|>


<commit_before>#!python3

import blackmamba as bm

################################################################
# Default Black Mamba external keyboard shortcuts registration #
################################################################

bm.register_default_key_commands()

################################################################
# Open/Run Quickly... ignore folders                           #
################################################################

# ''           : any parent folder
# '.'          : ~/Documents parent folder
# 'Development': ~/Documents/Development parent folder

bm.settings.OPEN_QUICKLY_IGNORE_FOLDERS = {
    '': ['.git'],
    '.': ['Pythonista', '.Trash', 'Examples',
          'site-packages-2', 'site-packages', 'stash_extensions'],
    'site-packages-3': ['blackmamba'],
    'Development': ['bm-pip-backup']
}

bm.settings.RUN_QUICKLY_IGNORE_FOLDERS = {
    '': ['.git'],
    '.': ['Pythonista', '.Trash', 'Examples',
          'site-packages-2', 'site-packages', 'stash_extensions'],
    'site-packages-3': ['blackmamba'],
    'Development': ['bm-pip-backup']
}

################################################################
# Analyzer                                                     #
################################################################

# Must be None or tuple ('W391', 'W293', ...)
bm.settings.ANALYZER_PEP8_IGNORE = None

# Max line length (E501)
# 127 is the GitHub editor max line length
bm.settings.ANALYZER_PEP8_MAX_LINE_LENGTH = 127

################################################################
# Unit tests                                                   #
################################################################

TESTS_HUD_DELAY = 1.0
TESTS_HIDE_CONSOLE = True

################################################################
# Custom keyboard shortcuts                                    #
################################################################

# Launch StaSh (= custom action title) via Cmd-Shift-S
if bm.ide.action_exists('StaSh'):
    def launch_stash():
        bm.ide.run_action('StaSh')

    bm.key_commands.register_key_command(
        'S',
        bm.uikit.UIKeyModifierCommand | bm.uikit.UIKeyModifierShift,
        launch_stash,
        'Launch StaSh')

# Or you can use run_script instead of action to launch StaSh
# if bm.ide.script_exists('launch_stash.py'):
#     def launch_stash():
#         bm.ide.run_script('launch_stash.py')
#
#     bm.key_commands.register_key_command(
#         'S',
#         bm.uikit.UIKeyModifierCommand | bm.uikit.UIKeyModifierShift,
#         launch_stash,
#         'Launch StaSh')
<commit_msg>Fixed tests settings<commit_after>#!python3

import blackmamba as bm

################################################################
# Default Black Mamba external keyboard shortcuts registration #
################################################################

bm.register_default_key_commands()

################################################################
# Open/Run Quickly... ignore folders                           #
################################################################

# ''           : any parent folder
# '.'          : ~/Documents parent folder
# 'Development': ~/Documents/Development parent folder

bm.settings.OPEN_QUICKLY_IGNORE_FOLDERS = {
    '': ['.git'],
    '.': ['Pythonista', '.Trash', 'Examples',
          'site-packages-2', 'site-packages', 'stash_extensions'],
    'site-packages-3': ['blackmamba'],
    'Development': ['bm-pip-backup']
}

bm.settings.RUN_QUICKLY_IGNORE_FOLDERS = {
    '': ['.git'],
    '.': ['Pythonista', '.Trash', 'Examples',
          'site-packages-2', 'site-packages', 'stash_extensions'],
    'site-packages-3': ['blackmamba'],
    'Development': ['bm-pip-backup']
}

################################################################
# Analyzer                                                     #
################################################################

# Must be None or tuple ('W391', 'W293', ...)
bm.settings.ANALYZER_PEP8_IGNORE = None

# Max line length (E501)
# 127 is the GitHub editor max line length
bm.settings.ANALYZER_PEP8_MAX_LINE_LENGTH = 127

################################################################
# Unit tests                                                   #
################################################################

bm.settings.TESTS_HUD_DELAY = 1.0
bm.settings.TESTS_HIDE_CONSOLE = True

################################################################
# Custom keyboard shortcuts                                    #
################################################################

# Launch StaSh (= custom action title) via Cmd-Shift-S
if bm.ide.action_exists('StaSh'):
    def launch_stash():
        bm.ide.run_action('StaSh')

    bm.key_commands.register_key_command(
        'S',
        bm.uikit.UIKeyModifierCommand | bm.uikit.UIKeyModifierShift,
        launch_stash,
        'Launch StaSh')

# Or you can use run_script instead of action to launch StaSh
# if bm.ide.script_exists('launch_stash.py'):
#     def launch_stash():
#         bm.ide.run_script('launch_stash.py')
#
#     bm.key_commands.register_key_command(
#         'S',
#         bm.uikit.UIKeyModifierCommand | bm.uikit.UIKeyModifierShift,
#         launch_stash,
#         'Launch StaSh')
<|endoftext|>


<commit_before>import time
import logging
import numpy as np
from math import ceil
from os.path import splitext

from analysis.plotting.plotting import plot_occupancy
from daq.readout import get_col_row_array_from_data_record_array, convert_data_array, data_array_from_data_dict_iterable, is_data_record, is_data_from_channel

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

local_configuration = {
    "cfg_name": 'threshold_baseline_tuning',  # the name of the new config with the tuning
    "occupancy_limit": 0,  # 0 will mask any pixel with occupancy greater than zero
    "disabled_pixels_limit": 0.01,  # in percent
    "repeat_tuning": False,
    "use_enable_mask": False,
    "triggers": 100000,
    "trig_count": 1,
    "col_span": [1, 80],
    "row_span": [1, 336],
    "timeout_no_data": 10
}

class ThresholdBaselineTuning(ScanBase):
    scan_id = "threshold_basline_tuning"

    def scan(self, cfg_name='noise_occ_tuning', occupancy_limit=0, disabled_pixels_limit=0.01, repeat_tuning=False, use_enable_mask=False, triggers=100000, trig_count=1, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, **kwargs):
        '''Masking pixels with occupancy above certain limit.

        Parameters
        ----------
        cfg_name : string
            File name of the configuration file. If None or not given, use default file name.
        occupancy_limit : float
            Occupancy limit which for each pixel. Any pixel above the limit the TDAC will be decreased (the lower TDAC value, the higher the threshold).
        disabled_pixels_limit : float
            Limit percentage of pixels, which will be disabled during tuning. Pixels will be disables when noisy and TDAC is 0 (highest possible threshold). Abort condition for baseline tuning.
        repeat_tuning : bool
            Repeat TDAC tuning each global threshold step until no noisy pixels occur. Usually not needed, default is disabled.
        use_enable_mask : bool
            Use enable mask for masking pixels.
        triggers : int
            Total number of triggers sent to FE. From 1 to 4294967295 (32-bit unsigned int).
        trig_count : int
            FE global register Trig_Count.
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.

        Note
        ----
        The total number of trigger is triggers * consecutive_lvl1.
        Please note that a high trigger rate leads to an effective lower threshold.
        '''
        # create restore point
        self.register.create_restore_point()
        self.trig_count = trig_count
        if trig_count == 0:
            consecutive_lvl1 = (2 ** self.register.get_global_register_objects(name=['Trig_Count'])[0].bitlength)
        else:
            consecutive_lvl1 = trig_count
        if occupancy_limit * triggers * consecutive_lvl1 < 1.0:
            logging.warning('Number of triggers too low for given occupancy limit. Any noise hit will lead to a masked pixel.')

        commands = []
        commands.extend(self.register.get_commands("confmode"))
        # TDAC
        tdac_median = np.median(self.register.get_pixel_register_value('TDAC'))
        tdac_max = 2 ** self.register.get_pixel_register_objects(name=['TDAC'])[0].bitlength - 1
        threshold_correction = 4 * ceil(tdac_max - tdac_median)
        if threshold_correction < 0.0:
            threshold_correction = 0.0
        pixel_reg = "TDAC"
        self.register.set_pixel_register_value(pixel_reg, tdac_max)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        pixel_reg = "Enable"
        if use_enable_mask:
            self.register.set_pixel_register_value(pixel_reg, np.logical_and(mask, self.register.get_pixel_register_value(pixel_reg)))
        else:
            self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate mask for Imon mask
        pixel_reg = "Imon"
        self.register.set_pixel_register_value(pixel_reg, 1)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
#         self.register.set_global_register_value("Trig_Lat", 232)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", trig_count)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["Trig_Count"]))
        # setting FE into runmode
        commands.extend(self.register.get_commands("runmode"))
        self.register_utils.send_commands(commands)

        vthin_alt_fine_max = 2 ** self.register.get_global_register_objects(name=["Vthin_AltFine"])[0].bitlength - 1
        if self.register.get_global_register_value("Vthin_AltFine") + threshold_correction > vthin_alt_fine_max:
            corrected_threshold = vthin_alt_fine_max
        else:
            corrected_threshold = self.register.get_global_register_value("Vthin_AltFine") + threshold_correction

        disabled_pixels_limit_cnt = int(disabled_pixels_limit * 336 * 80)
        diabled_pixels = 0

        for reg_val in range(int(corrected_threshold), -1, -1):
            self.register.create_restore_point(name=str(reg_val))
            self.stop_thread_event.clear()
            logging.info('Scanning Vthin_AltFine %d' % reg_val)
            commands = []
            commands.extend(self.register.get_commands("confmode"))
            self.register.set_global_register_value("Vthin_AltFine", reg_val)  # set number of consecutive triggers
            commands.extend(self.register.get_commands("wrregister", name=["Vthin_AltFine"]))
            # setting FE into runmode
            commands.extend(self.register.get_commands("runmode"))
            self.register_utils.send_commands(commands)

            while True:

                self.col_arr = np.array([], dtype=np.dtype('>u1'))
                self.row_arr = np.array([], dtype=np.dtype('>u1'))

                with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_id) as raw_data_file:
                    self.readout.start()

                    # preload command
                    command_delay = 500  # <100kHz
                    lvl1_command = self.register.get_commands("lv1")[0] + self.register.get_commands("zeros", length=command_delay)[0]
                    commnd_lenght = lvl1_command.length()
                    logging.info('Estimated scan time: %ds' % int(commnd_lenght * 25 * (10 ** -9) * triggers))
                    logging.info('Please stand by...')
                    self.register_utils.send_command(lvl1_command, repeat=triggers, wait_for_finish=False, set_length=True, clear_memory=False)

                    wait_for_first_data = False
                    last_iteration = time.time()
                    saw_no_data_at_time = last_iteration
                    saw_data_at_time = last_iteration
                    no_data_at_time = last_iteration
                    while not self.stop_thread_event.wait(self.readout.readout_interval):
                        last_iteration = time.time()
                        try:
                            data = (self.readout.data.popleft(), )
                            raw_data_file.append(data)
                            col_arr_tmp, row_arr_tmp = convert_data_array(data_array_from_data_dict_iterable(data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
                            self.col_arr = np.concatenate((self.col_arr, col_arr_tmp))
                            self.row_arr = np.concatenate((self.row_arr, row_arr_tmp))
                        except IndexError:  # no data
                            no_data_at_time = last_iteration
                            if self.register_utils.is_ready:
                                self.stop_thread_event.set()
                                logging.info('Finished sending %d triggers' % triggers)
                            elif wait_for_first_data is False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                                logging.info('Reached no data timeout. Stopping Scan...')
                                self.stop_thread_event.set()
                            elif wait_for_first_data is False:
                                saw_no_data_at_time = no_data_at_time
                            elif self.reaout_utils.is_ready:
                                self.stop_thread_event.set()

                            continue

                        saw_data_at_time = last_iteration

                        if wait_for_first_data is True:
                            logging.info('Taking data...')
                            wait_for_first_data = False

                    self.readout.stop()

                    occ_hist, _, _ = np.histogram2d(self.col_arr, self.row_arr, bins=(80, 336), range=[[1, 80], [1, 336]])
                    self.occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
                    # noisy pixels are set to 1
                    self.occ_mask[occ_hist > occupancy_limit * triggers * consecutive_lvl1] = 1
                    plot_occupancy(occ_hist.T, title='Occupancy', filename=scan.scan_data_filename + '_noise_occ_' + str(reg_val) + '.pdf')

                    tdac_reg = self.register.get_pixel_register_value('TDAC')
                    decrease_pixel_mask = np.logical_and(self.occ_mask > 0, tdac_reg > 0)
                    disable_pixel_mask = np.logical_and(self.occ_mask > 0, tdac_reg == 0)
                    enable_reg = self.register.get_pixel_register_value('Enable')
                    enable_mask = np.logical_and(enable_reg, self.register_utils.invert_pixel_mask(disable_pixel_mask))
                    diabled_pixels += disable_pixel_mask.sum()
                    plot_occupancy(tdac_reg.T, title='TDAC', filename=scan.scan_data_filename + '_TDAC_' + str(reg_val) + '.pdf')
                    if diabled_pixels > disabled_pixels_limit_cnt:
                        logging.info('Limit of disabled pixels reached: %d (limit %d)... stopping scan' % (diabled_pixels, disabled_pixels_limit_cnt))
                        self.register.restore(name=str(reg_val))
                        break
                    else:
                        logging.info('Tuned %d pixel(s)' % (decrease_pixel_mask.sum(),))
                        logging.info('Disabled %d pixel(s)' % (diabled_pixels,))
                        tdac_reg[decrease_pixel_mask] -= 1  # TODO
                        self.register.set_pixel_register_value('TDAC', tdac_reg)
                        self.register.set_pixel_register_value('Enable', enable_mask)
                        commands = []
                        commands.extend(self.register.get_commands("confmode"))
                        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name='TDAC'))
                        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name='Enable'))
                        commands.extend(self.register.get_commands("runmode"))
                        self.register_utils.send_commands(commands)
                        if not repeat_tuning or self.occ_mask.sum() == 0:
                            self.register.clear_restore_points(name=str(reg_val))
                            break
                        else:
                            logging.info('Found noisy pixels... repeat tuning step for Vthin_AltFine %d' % (reg_val,))

            if diabled_pixels > disabled_pixels_limit_cnt:
                last_good_threshold = self.register.get_global_register_value("Vthin_AltFine")
                last_good_tdac = self.register.get_pixel_register_value('TDAC')
                last_good_enable_mask = self.register.get_pixel_register_value('Enable')
                self.register.restore()
                self.register.set_global_register_value("Vthin_AltFine", last_good_threshold)
                self.register.set_pixel_register_value('TDAC', last_good_tdac)
                self.register.set_pixel_register_value('Enable', last_good_enable_mask)
                self.register.save_configuration(cfg_name if cfg_name else (splitext(self.device_configuration["configuration_file"])[0] + '_' + self.scan_id))
                break

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename, analyzed_data_file=output_file, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.trig_count)
            analyze_raw_data.create_source_scan_hist = True
#             analyze_raw_data.create_hit_table = True
#             analyze_raw_data.interpreter.debug_events(0, 0, True)  # events to be printed onto the console for debugging, usually deactivated
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()
            plot_occupancy(self.occ_mask.T, title='Noisy Pixels', z_max=1, filename=analyze_raw_data.output_pdf)

if __name__ == "__main__":
    import configuration
    scan = ThresholdBaselineTuning(**configuration.default_configuration)
    scan.start(use_thread=False, **local_configuration)
    scan.stop()
#     scan.analyze()
#    scan.register.save_configuration()
<commit_msg>ENH: better logging<commit_after>import time
import logging
import numpy as np
from math import ceil
from os.path import splitext

from analysis.plotting.plotting import plot_occupancy
from daq.readout import get_col_row_array_from_data_record_array, convert_data_array, data_array_from_data_dict_iterable, is_data_record, is_data_from_channel

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

local_configuration = {
    "cfg_name": 'threshold_baseline_tuning',  # the name of the new config with the tuning
    "occupancy_limit": 0,  # 0 will mask any pixel with occupancy greater than zero
    "disabled_pixels_limit": 0.01,  # in percent
    "repeat_tuning": False,
    "use_enable_mask": False,
    "triggers": 100000,
    "trig_count": 1,
    "col_span": [1, 80],
    "row_span": [1, 336],
    "timeout_no_data": 10
}

class ThresholdBaselineTuning(ScanBase):
    scan_id = "threshold_basline_tuning"

    def scan(self, cfg_name='noise_occ_tuning', occupancy_limit=0, disabled_pixels_limit=0.01, repeat_tuning=False, use_enable_mask=False, triggers=100000, trig_count=1, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10, **kwargs):
        '''Masking pixels with occupancy above certain limit.

        Parameters
        ----------
        cfg_name : string
            File name of the configuration file. If None or not given, use default file name.
        occupancy_limit : float
            Occupancy limit which for each pixel. Any pixel above the limit the TDAC will be decreased (the lower TDAC value, the higher the threshold).
        disabled_pixels_limit : float
            Limit percentage of pixels, which will be disabled during tuning. Pixels will be disables when noisy and TDAC is 0 (highest possible threshold). Abort condition for baseline tuning.
        repeat_tuning : bool
            Repeat TDAC tuning each global threshold step until no noisy pixels occur. Usually not needed, default is disabled.
        use_enable_mask : bool
            Use enable mask for masking pixels.
        triggers : int
            Total number of triggers sent to FE. From 1 to 4294967295 (32-bit unsigned int).
        trig_count : int
            FE global register Trig_Count.
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.

        Note
        ----
        The total number of trigger is triggers * consecutive_lvl1.
        Please note that a high trigger rate leads to an effective lower threshold.
        '''
        # create restore point
        self.register.create_restore_point()
        self.trig_count = trig_count
        if trig_count == 0:
            consecutive_lvl1 = (2 ** self.register.get_global_register_objects(name=['Trig_Count'])[0].bitlength)
        else:
            consecutive_lvl1 = trig_count
        if occupancy_limit * triggers * consecutive_lvl1 < 1.0:
            logging.warning('Number of triggers too low for given occupancy limit. Any noise hit will lead to a masked pixel.')

        commands = []
        commands.extend(self.register.get_commands("confmode"))
        # TDAC
        tdac_median = np.median(self.register.get_pixel_register_value('TDAC'))
        tdac_max = 2 ** self.register.get_pixel_register_objects(name=['TDAC'])[0].bitlength - 1
        threshold_correction = 4 * ceil(tdac_max - tdac_median)
        if threshold_correction < 0.0:
            threshold_correction = 0.0
        pixel_reg = "TDAC"
        self.register.set_pixel_register_value(pixel_reg, tdac_max)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        pixel_reg = "Enable"
        if use_enable_mask:
            self.register.set_pixel_register_value(pixel_reg, np.logical_and(mask, self.register.get_pixel_register_value(pixel_reg)))
        else:
            self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate mask for Imon mask
        pixel_reg = "Imon"
        self.register.set_pixel_register_value(pixel_reg, 1)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
#         self.register.set_global_register_value("Trig_Lat", 232)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", trig_count)  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["Trig_Count"]))
        # setting FE into runmode
        commands.extend(self.register.get_commands("runmode"))
        self.register_utils.send_commands(commands)

        vthin_alt_fine_max = 2 ** self.register.get_global_register_objects(name=["Vthin_AltFine"])[0].bitlength - 1
        if self.register.get_global_register_value("Vthin_AltFine") + threshold_correction > vthin_alt_fine_max:
            corrected_threshold = vthin_alt_fine_max
        else:
            corrected_threshold = self.register.get_global_register_value("Vthin_AltFine") + threshold_correction

        disabled_pixels_limit_cnt = int(disabled_pixels_limit * 336 * 80)
        diabled_pixels = 0

        for reg_val in range(int(corrected_threshold), -1, -1):
            self.register.create_restore_point(name=str(reg_val))
            self.stop_thread_event.clear()
            logging.info('Scanning Vthin_AltFine %d' % reg_val)
            commands = []
            commands.extend(self.register.get_commands("confmode"))
            self.register.set_global_register_value("Vthin_AltFine", reg_val)  # set number of consecutive triggers
            commands.extend(self.register.get_commands("wrregister", name=["Vthin_AltFine"]))
            # setting FE into runmode
            commands.extend(self.register.get_commands("runmode"))
            self.register_utils.send_commands(commands)

            while True:

                self.col_arr = np.array([], dtype=np.dtype('>u1'))
                self.row_arr = np.array([], dtype=np.dtype('>u1'))

                with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_id) as raw_data_file:
                    self.readout.start()

                    # preload command
                    command_delay = 500  # <100kHz
                    lvl1_command = self.register.get_commands("lv1")[0] + self.register.get_commands("zeros", length=command_delay)[0]
                    commnd_lenght = lvl1_command.length()
                    logging.info('Estimated scan time: %ds' % int(commnd_lenght * 25 * (10 ** -9) * triggers))
                    logging.info('Please stand by...')
                    self.register_utils.send_command(lvl1_command, repeat=triggers, wait_for_finish=False, set_length=True, clear_memory=False)

                    wait_for_first_data = False
                    last_iteration = time.time()
                    saw_no_data_at_time = last_iteration
                    saw_data_at_time = last_iteration
                    no_data_at_time = last_iteration
                    while not self.stop_thread_event.wait(self.readout.readout_interval):
                        last_iteration = time.time()
                        try:
                            data = (self.readout.data.popleft(), )
                            raw_data_file.append(data)
                            col_arr_tmp, row_arr_tmp = convert_data_array(data_array_from_data_dict_iterable(data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
                            self.col_arr = np.concatenate((self.col_arr, col_arr_tmp))
                            self.row_arr = np.concatenate((self.row_arr, row_arr_tmp))
                        except IndexError:  # no data
                            no_data_at_time = last_iteration
                            if self.register_utils.is_ready:
                                self.stop_thread_event.set()
                                logging.info('Finished sending %d triggers' % triggers)
                            elif wait_for_first_data is False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                                logging.info('Reached no data timeout. Stopping Scan...')
                                self.stop_thread_event.set()
                            elif wait_for_first_data is False:
                                saw_no_data_at_time = no_data_at_time
                            elif self.reaout_utils.is_ready:
                                self.stop_thread_event.set()

                            continue

                        saw_data_at_time = last_iteration

                        if wait_for_first_data is True:
                            logging.info('Taking data...')
                            wait_for_first_data = False

                    self.readout.stop()

                    occ_hist, _, _ = np.histogram2d(self.col_arr, self.row_arr, bins=(80, 336), range=[[1, 80], [1, 336]])
                    self.occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
                    # noisy pixels are set to 1
                    self.occ_mask[occ_hist > occupancy_limit * triggers * consecutive_lvl1] = 1
                    plot_occupancy(occ_hist.T, title='Occupancy', filename=scan.scan_data_filename + '_noise_occ_' + str(reg_val) + '.pdf')

                    tdac_reg = self.register.get_pixel_register_value('TDAC')
                    decrease_pixel_mask = np.logical_and(self.occ_mask > 0, tdac_reg > 0)
                    disable_pixel_mask = np.logical_and(self.occ_mask > 0, tdac_reg == 0)
                    enable_reg = self.register.get_pixel_register_value('Enable')
                    enable_mask = np.logical_and(enable_reg, self.register_utils.invert_pixel_mask(disable_pixel_mask))
                    diabled_pixels += disable_pixel_mask.sum()
                    plot_occupancy(tdac_reg.T, title='TDAC', filename=scan.scan_data_filename + '_TDAC_' + str(reg_val) + '.pdf')
                    if diabled_pixels > disabled_pixels_limit_cnt:
                        logging.info('Limit of disabled pixels reached: %d (limit %d)... stopping scan' % (diabled_pixels, disabled_pixels_limit_cnt))
                        self.register.restore(name=str(reg_val))
                        break
                    else:
                        logging.info('Increasing threshold of %d pixel(s)' % (decrease_pixel_mask.sum(),))
                        logging.info('Disabling %d pixel(s), total number of disabled pixel(s): %d' % (disable_pixel_mask.sum(), diabled_pixels))
                        tdac_reg[decrease_pixel_mask] -= 1  # TODO
                        self.register.set_pixel_register_value('TDAC', tdac_reg)
                        self.register.set_pixel_register_value('Enable', enable_mask)
                        commands = []
                        commands.extend(self.register.get_commands("confmode"))
                        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name='TDAC'))
                        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name='Enable'))
                        commands.extend(self.register.get_commands("runmode"))
                        self.register_utils.send_commands(commands)
                        if not repeat_tuning or self.occ_mask.sum() == 0:
                            self.register.clear_restore_points(name=str(reg_val))
                            break
                        else:
                            logging.info('Found noisy pixels... repeat tuning step for Vthin_AltFine %d' % (reg_val,))

            if diabled_pixels > disabled_pixels_limit_cnt:
                last_good_threshold = self.register.get_global_register_value("Vthin_AltFine")
                last_good_tdac = self.register.get_pixel_register_value('TDAC')
                last_good_enable_mask = self.register.get_pixel_register_value('Enable')
                self.register.restore()
                self.register.set_global_register_value("Vthin_AltFine", last_good_threshold)
                self.register.set_pixel_register_value('TDAC', last_good_tdac)
                self.register.set_pixel_register_value('Enable', last_good_enable_mask)
                self.register.save_configuration(cfg_name if cfg_name else (splitext(self.device_configuration["configuration_file"])[0] + '_' + self.scan_id))
                break

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename, analyzed_data_file=output_file, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.trig_count)
            analyze_raw_data.create_source_scan_hist = True
#             analyze_raw_data.create_hit_table = True
#             analyze_raw_data.interpreter.debug_events(0, 0, True)  # events to be printed onto the console for debugging, usually deactivated
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()
            plot_occupancy(self.occ_mask.T, title='Noisy Pixels', z_max=1, filename=analyze_raw_data.output_pdf)

if __name__ == "__main__":
    import configuration
    scan = ThresholdBaselineTuning(**configuration.default_configuration)
    scan.start(use_thread=False, **local_configuration)
    scan.stop()
#     scan.analyze()
#    scan.register.save_configuration()
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
#
#
#    Copyright (C) 2011,2013 Michael Telahun Makonnen <mmakonnen@gmail.com>.
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#

from openerp import models, fields

GENDER_SELECTION = [('m', 'M'),
                    ('f', 'F')]

class HrChildren(models.Model):
    _name = 'hr.employee.children'
    _description = 'HR Employee Children'

    name = fields.Char("Name", required=True)
    date_of_birth = fields.Date("Date of Birth", oldname='dob')
    employee_id = fields.Many2one('hr.employee', "Employee")
    gender = fields.Selection(selection=GENDER_SELECTION, string='Gender')
<commit_msg>[IMP][hr_family] Use the same selection like employee<commit_after># -*- coding:utf-8 -*-
#
#
#    Copyright (C) 2011,2013 Michael Telahun Makonnen <mmakonnen@gmail.com>.
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#

from openerp import models, fields

GENDER_SELECTION = [('male', 'Male'),
                    ('female', 'Female')]

class HrChildren(models.Model):
    _name = 'hr.employee.children'
    _description = 'HR Employee Children'

    name = fields.Char("Name", required=True)
    date_of_birth = fields.Date("Date of Birth", oldname='dob')
    employee_id = fields.Many2one('hr.employee', "Employee")
    gender = fields.Selection(selection=GENDER_SELECTION, string='Gender')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Oct 20 17:40:07 2015

@author: casimp
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re
import os
import fabio
import h5py
import numpy as np
import pyFAI
import sys

from pyxe.data_io import dim_fill, extract_fnames, dimension_fill_pixium10
from pyxe.peak_analysis import PeakAnalysis
from pyxpb.detectors import MonoDetector

class Mono(PeakAnalysis):

    def __init__(self, folder, co_ords, detector, wavelength=None,
                 f_ext='.edf', progress=True, npt_rad=1024, npt_az=36,
                 az_range=(-180, 180)):
        """
        Takes a folder containing image files from area detectors and cakes
        the data while associating it with spatial information. The caked data
        can then be analysed (peak_fit/strain calculations).

        Args:
            folder (str): Folder containing the image files for analysis
            co_ords (ndarray): 1D/2D/3D numpy array containing data co-ords
            detector (tuple, object): pyFAI detector object or Fit2D params:
                (sample_to_detector (mm), centre_x (pix), centre_y (pix),
                tilt (deg), tilt_plane (deg), pixel_size_x (micron),
                pixel_size_y (micron)).
            wavelength (float): Wavelength (nm) (if not defined in pyFAI file)
            f_ext (str): File extension of image files
            progress (bool): Live progress bar
            npt_rad (int): Number of radial bins (approx. half detect width)
            npt_az (int): Number of azimuthal wedges
            az_range (tuple): Azimtuhal range (deg) - default (-180, 180).
                Note that the 0deg is at eastern edge of circle.
        """
        fname = '{}.h5'.format(os.path.split(folder)[1])
        self.fpath = os.path.join(folder, fname)

        error = 'Azimuthal range must be less than or equal to 360deg'
        assert abs(np.max(az_range) - np.min(az_range)) <= 360, error

        # Allow for use of pyFAI or Fit2D detector params
        # Currently checking if folder string - perhaps not the best method!

        if isinstance(detector, pyFAI.azimuthalIntegrator.AzimuthalIntegrator):
            ai = detector
            if wavelength is None:
                ai.get_wavelength()
            else:
                ai.set_wavelength(wavelength)
        else:
            ai = pyFAI.AzimuthalIntegrator()
            ai.setFit2D(*detector)
            ai.set_wavelength(wavelength)
        fnames = extract_fnames(folder, f_ext)

        error = ('Number of positions not equal to number of files (pos = %s,'
                 ' files = %s)' % (co_ords.shape[0], len(fnames)))
        assert co_ords.shape[0] == len(fnames), error
        (self.d1, self.d2, self.d3), self.dims = dim_fill(co_ords)
        self.ndim = len(self.dims)
        self.I = np.nan * np.ones((co_ords.shape[0], npt_az, npt_rad))

        print('\nLoading files and carrying out azimuthal integration:\n')
        for fidx, fname in enumerate(fnames):
            img = fabio.open(os.path.join(folder, fname)).data
            npt_rad, npt_az = int(npt_rad), int(npt_az)
            I, q_, phi = ai.integrate2d(img, npt_rad=npt_rad, npt_azim=npt_az,
                                        azimuth_range=az_range, unit='q_A^-1')
            self.I[fidx] = I
            if progress:
                sys.stdout.write('\rProgress: [{0:20s}] {1:.0f}%'.format('#' *
                                 int(20*(fidx + 1) / len(fnames)),
                                 100*((fidx + 1)/len(fnames))))
                sys.stdout.flush()

        # Create az_slice 'specific' q values - to work with edxd data
        self.q = np.repeat(q_[None, :], npt_az, axis=0)
        self.phi = phi * np.pi / 180
        self.analysis_state = 'integrated'
        # Temporary - extract from ai!
        self.detector = MonoDetector((2000,2000), 0.1, 1000, 100, 1)

class MonoI12_old(PeakAnalysis):

    def __init__(self, fpath, rpath, detector=None):
        """ Extract useful data from pre-processed .nxs file.

        *** Un-tested ***

        Analysis of pre-processed (azimuthally integrated) data from the I12
        beamline. Data file should approximate to the .nxs from the EDXD
        detector but still untested. Should re-arrange and scrape pre-processed
        data to correct form.

        Args:
            fpath (str): Path to processed datafile (.nxs)
            rpath (str): Path to raw datafile (.nxs)
            detector (tuple, object): pyFAI detector object or Fit2D params:
                (sample_to_detector (mm), centre_x (pix), centre_y (pix),
                tilt (deg), tilt_plane (deg), pixel_size_x (micron),
                pixel_size_y (micron)).
        """
        self.fpath = fpath
        f = h5py.File(fpath, 'r')
        raw = h5py.File(rpath, 'r')
        # Use scan command to find number of dimensions and the order in which
        # they were acquired. Important/useful for plotting!
        scan_command = raw['entry1/scan_command'][()][0]
        dims = re.findall(b'ss2_\w+', scan_command) # valid?
        self.ndim = len(dims)
        all_dims = [b'ss2_x', b'ss2_y', b'ss2_z']
        dims = dims + [dim for dim in all_dims if dim not in dims]
        co_ords = []
        for dim in dims:
            co_ords.append(dimension_fill_pixium10(raw, dim.decode("utf-8")))
        self.d1, self.d2, self.d3 = co_ords

        q = f['entry/result/q'][()] # will be incorrect

        self.I = f['entry/result/data'][()] # will be incorrect
        self.phi = np.pi * f['entry/result/azimuthal angle (degrees)'][()] / 180 # will be incorrect
        self.q = np.repeat(q[None, :], self.phi.size, axis=0)
        self.analysis_state = 'integrated'
        ##### FIX THIS!
        if detector is None:
            # scrape from file..?
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)
        else:
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)

class MonoI12(PeakAnalysis):

    def __init__(self, fpath, detector=None):
        """ Extract useful data from pre-processed .nxs file.

        *** Un-tested ***

        Analysis of pre-processed (azimuthally integrated) data from the I12
        beamline. Data file should approximate to the .nxs from the EDXD
        detector but still untested. Should re-arrange and scrape pre-processed
        data to correct form.

        Args:
            fpath (str): Path to processed datafile (.nxs)
            detector (tuple, object): pyFAI detector object or Fit2D params:
                (sample_to_detector (mm), centre_x (pix), centre_y (pix),
                tilt (deg), tilt_plane (deg), pixel_size_x (micron),
                pixel_size_y (micron)).
        """
        self.fpath = fpath
        f = h5py.File(fpath, 'r')
        self.I = f['entry/result/data'][()]
        self.phi = np.pi * f['entry/result/azimuthal angle (degrees)'][()] / 180
        q = f['entry/result/q'][()]
        self.q = np.repeat(q[None, :], self.phi.size, axis=0)

        self.ndim = len(self.I.shape[:-2])
        dims = [i for i in list(f['entry/result/'].keys()) if i[:2] == 'ss']
        self.d1, self.d2, self.d3 = None, None, None
        try:
            self.d1 = f['entry/result/{}'.format(dims[0])][()]
            self.d2 = f['entry/result/{}'.format(dims[1])][()]
            self.d3 = f['entry/result/{}'.format(dims[2])][()]
        except IndexError:
            pass

        # This will not work with 3D data (which I'm yet to come across!)
        if self.d1 is not None and len(self.d1.shape) != self.ndim:
            self.d1, self.d2 = np.meshgrid(self.d1, self.d2)

        self.analysis_state = 'integrated'
        ##### FIX THIS!
        if detector is None:
            # scrape from file..?

            self.detector = MonoDetector((2043, 2043), 0.1, 1000, 100, 1)
        else:
            self.detector = MonoDetector((2043, 2043), 0.1, 1000, 100, 1)

<commit_msg>Cleaned the monochromatic I12 analysis - more flexible.<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Oct 20 17:40:07 2015

@author: casimp
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re
import os
import fabio
import h5py
import numpy as np
import pyFAI
import sys
import scipy.io as io

from pyxe.data_io import dim_fill, extract_fnames, dimension_fill_pixium10
from pyxe.peak_analysis import PeakAnalysis
from pyxpb.detectors import MonoDetector

class Mono(PeakAnalysis):

    def __init__(self, folder, co_ords, detector, wavelength=None,
                 f_ext='.edf', progress=True, npt_rad=1024, npt_az=36,
                 az_range=(-180, 180)):
        """
        Takes a folder containing image files from area detectors and cakes
        the data while associating it with spatial information. The caked data
        can then be analysed (peak_fit/strain calculations).

        Args:
            folder (str): Folder containing the image files for analysis
            co_ords (ndarray): 1D/2D/3D numpy array containing data co-ords
            detector (tuple, object): pyFAI detector object or Fit2D params:
                (sample_to_detector (mm), centre_x (pix), centre_y (pix),
                tilt (deg), tilt_plane (deg), pixel_size_x (micron),
                pixel_size_y (micron)).
            wavelength (float): Wavelength (nm) (if not defined in pyFAI file)
            f_ext (str): File extension of image files
            progress (bool): Live progress bar
            npt_rad (int): Number of radial bins (approx. half detect width)
            npt_az (int): Number of azimuthal wedges
            az_range (tuple): Azimtuhal range (deg) - default (-180, 180).
                Note that the 0deg is at eastern edge of circle.
        """
        fname = '{}.h5'.format(os.path.split(folder)[1])
        self.fpath = os.path.join(folder, fname)

        error = 'Azimuthal range must be less than or equal to 360deg'
        assert abs(np.max(az_range) - np.min(az_range)) <= 360, error

        # Allow for use of pyFAI or Fit2D detector params
        # Currently checking if folder string - perhaps not the best method!

        if isinstance(detector, pyFAI.azimuthalIntegrator.AzimuthalIntegrator):
            ai = detector
            if wavelength is None:
                ai.get_wavelength()
            else:
                ai.set_wavelength(wavelength)
        else:
            ai = pyFAI.AzimuthalIntegrator()
            ai.setFit2D(*detector)
            ai.set_wavelength(wavelength)
        fnames = extract_fnames(folder, f_ext)

        error = ('Number of positions not equal to number of files (pos = %s,'
                 ' files = %s)' % (co_ords.shape[0], len(fnames)))
        assert co_ords.shape[0] == len(fnames), error
        (self.d1, self.d2, self.d3), self.dims = dim_fill(co_ords)
        self.T = None
        self.ndim = len(self.dims)
        self.I = np.nan * np.ones((co_ords.shape[0], npt_az, npt_rad))

        print('\nLoading files and carrying out azimuthal integration:\n')
        for fidx, fname in enumerate(fnames):
            img = fabio.open(os.path.join(folder, fname)).data
            npt_rad, npt_az = int(npt_rad), int(npt_az)
            I, q_, phi = ai.integrate2d(img, npt_rad=npt_rad, npt_azim=npt_az,
                                        azimuth_range=az_range, unit='q_A^-1')
            self.I[fidx] = I
            if progress:
                sys.stdout.write('\rProgress: [{0:20s}] {1:.0f}%'.format('#' *
                                 int(20*(fidx + 1) / len(fnames)),
                                 100*((fidx + 1)/len(fnames))))
                sys.stdout.flush()

        # Create az_slice 'specific' q values - to work with edxd data
        self.q = np.repeat(q_[None, :], npt_az, axis=0)
        self.phi = phi * np.pi / 180
        self.analysis_state = 'integrated'
        # Temporary - extract from ai!
        self.detector = MonoDetector((2000,2000), 0.1, 1000, 100, 1)

class MonoI12(PeakAnalysis):

    def __init__(self, fpath,
                 d1='entry/result/ss1_x',
                 d2='entry/result/ss1_y', d3=None,
                 q='entry/result/q',
                 I='entry/result/data',
                 phi='entry/result/azimuthal angle (degrees)',
                 detector=None):

        """ Extract useful data from pre-processed .nxs file.

        Analysis of pre-processed (azimuthally integrated) data from the I12
        beamline. Data file should approximate to the .nxs from the EDXD
        detector but still untested. Should re-arrange and scrape pre-processed
        data to correct form.

        Args:
            fpath (str): Path to processed datafile (.nxs)
            d1, d2, d3 (str): Path within .nxs file to co-ords
            q, I, phi (str): Path within .nxs file to q, intensity and phi
            detector (tuple, object): pyFAI detector object or Fit2D params:
                (sample_to_detector (mm), centre_x (pix), centre_y (pix),
                tilt (deg), tilt_plane (deg), pixel_size_x (micron),
                pixel_size_y (micron)).
        """
        self.fpath = fpath
        f = h5py.File(fpath, 'r')

        self.d1 = f[d1][()]
        self.d2 = None if d2 == None else f[d2][()]
        self.d3 = None if d3 == None else f[d3][()]
        self.ndim = sum(x is not None for x in [self.d1, self.d2, self.d3])

        q = f[q][()]
        self.I = f[I][()]
        self.phi = np.pi * f[phi][()] / 180
        self.q = np.repeat(q[None, :], self.phi.size, axis=0)
        self.analysis_state = 'integrated'
        ##### FIX THIS!
        if detector is None:
            # scrape from file..?
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)
        else:
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)

def import_from_mat(fpath):
    mat = io.loadmat(fpath, squeeze_me=True)

    counts,unc,exposure,epoch,samplez,sampley,mondio,fnames = [],[],[],[],[],[],[],[]
    for scan in range(len(mat['w'])):
        try:
            q = mat['w'][scan][()][0]
            _counts = mat['w'][scan][()][1]
            _unc = mat['w'][scan][()][2] #error on pixel?
            _exposure = mat['w'][scan][()][3]
            _epoch = mat['w'][scan][()][4]
            _laserz = mat['w'][scan][()][5] #z laser-sample separation - not varied
            _samplez = mat['w'][scan][()][6]
            #_samplez = mat['w'][scan][()][7] #ignore, same as above
            _sampley = mat['w'][scan][()][8] #pp01
            _fnames = mat['w'][scan][()][9]
            _mondio = mat['w'][scan][()][10] #region of interest on
        except:
            pass
            #print('Could not get scan %i'%scan)
        else:
            counts.append(_counts)
            unc.append(_unc)
            exposure.append(_exposure)
            epoch.append(_epoch)
            samplez.append(_samplez)
            sampley.append(_sampley)
            mondio.append(_mondio)
            fnames.append(_fnames)

    counts = np.dstack(counts)
    counts = np.swapaxes(counts, 0, 2)
    unc = np.dstack(unc)
    unc = np.swapaxes(unc, 0, 2)
    exposure = np.hstack(exposure)
    epoch = np.hstack(epoch)
    samplez = np.hstack(samplez)
    sampley = np.hstack(sampley)
    mondio = np.hstack(mondio)
    fnames = np.hstack(fnames)
    del mat

    return dict(q=q,counts=counts,unc=unc,exposure=exposure,epoch=epoch,
                samplez=samplez,sampley=sampley,mondio=mondio,fnames=fnames)

class MonoESRF_mat(PeakAnalysis):

    def __init__(self, fpath, detector=None):
        data = import_from_mat(fpath)
        self.ndim = 2
        self.d1, self. d2, self.d3 = data['samplez'], data['sampley'], None
        self.T = None
        q, self.I = data['q'], data['counts']
        self.phi = np.linspace(-np.pi, np.pi, 36)
        self.q = np.repeat(q[None, :], self.phi.size, axis=0)

        self.analysis_state = 'integrated'
        ##### FIX THIS!
        if detector is None:
            # scrape from file..?
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)
        else:
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)

class MonoI12_old(PeakAnalysis):

    def __init__(self, fpath, rpath, detector=None):
        """ Extract useful data from pre-processed .nxs file.

        *** Un-tested ***

        Analysis of pre-processed (azimuthally integrated) data from the I12
        beamline. Data file should approximate to the .nxs from the EDXD
        detector but still untested. Should re-arrange and scrape pre-processed
        data to correct form.

        Args:
            fpath (str): Path to processed datafile (.nxs)
            rpath (str): Path to raw datafile (.nxs)
            detector (tuple, object): pyFAI detector object or Fit2D params:
                (sample_to_detector (mm), centre_x (pix), centre_y (pix),
                tilt (deg), tilt_plane (deg), pixel_size_x (micron),
                pixel_size_y (micron)).
        """
        self.fpath = fpath
        f = h5py.File(fpath, 'r')
        raw = h5py.File(rpath, 'r')
        # Use scan command to find number of dimensions and the order in which
        # they were acquired. Important/useful for plotting!
        scan_command = raw['entry1/scan_command'][()][0]
        dims = re.findall(b'ss2_\w+', scan_command) # valid?
        self.ndim = len(dims)
        all_dims = [b'ss2_x', b'ss2_y', b'ss2_z']
        dims = dims + [dim for dim in all_dims if dim not in dims]
        co_ords = []
        for dim in dims:
            co_ords.append(dimension_fill_pixium10(raw, dim.decode("utf-8")))
        self.d1, self.d2, self.d3 = co_ords
        self.T = None

        q = f['entry/result/q'][()] # will be incorrect

        self.I = f['entry/result/data'][()] # will be incorrect
        self.phi = np.pi * f['entry/result/azimuthal angle (degrees)'][()] / 180 # will be incorrect
        self.q = np.repeat(q[None, :], self.phi.size, axis=0)
        self.analysis_state = 'integrated'
        ##### FIX THIS!
        if detector is None:
            # scrape from file..?
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)
        else:
            self.detector = MonoDetector((2000, 2000), 0.1, 1000, 100, 1)

<|endoftext|>