<commit_before>from citrination_client import CitrinationClient
from os import environ
import os
from json import loads
from pypif.obj.system import System
from pypif.pif import dump
import random
import string

def _almost_equal(test_value, reference_value, tolerance=1.0e-9):
    """
    Numerical equality with a tolerance
    """
    return abs(test_value - reference_value) < tolerance

class TestClient():

    @classmethod
    def setup_class(self):
        self.client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        # Append dataset name with random string because one user can't have more than
        # one dataset with the same name
        random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
        dataset_name = "Tutorial dataset " + random_string
        self.set_id = loads(self.client.create_data_set(name=dataset_name, description="Dataset for tutorial", share=0).content.decode('utf-8'))['id']
        self.test_file_root = './citrination_client/tests/test_files/'

    def get_test_file_hierarchy_count(self):
        test_dir = self.test_file_root
        return sum([len(files) for r, d, files in os.walk(test_dir)])

    def test_start_client(self):
        assert self.client is not None

    def test_upload_pif(self):
        pif = System()
        pif.id = 0

        with open("tmp.json", "w") as fp:
            dump(pif, fp)
        response = loads(self.client.upload_file("tmp.json", self.set_id))
        assert response["message"] == "Upload is complete."

    def test_file_listing(self):
        src_path = self.test_file_root + "keys_and_values.json"
        dest_path = "test_file_list.json"
        self.client.upload(self.set_id, src_path, dest_path)
        listed_files = self.client.list_files(self.set_id, dest_path)["files"]
        assert len(listed_files) == 1
        assert listed_files[0] == dest_path

    def test_upload_directory(self):
        count_to_add = self.get_test_file_hierarchy_count()
        src_path = self.test_file_root
        dest_path = "test_directory_upload/"
        before_count = self.client.matched_file_count(self.set_id)
        self.client.upload(self.set_id, src_path, dest_path)
        after_count = self.client.matched_file_count(self.set_id)
        assert after_count == (before_count + count_to_add)

    @staticmethod
    def _test_prediction_values(prediction):
        """
        Assertions for the test_predict and test_predict_distribution methods
        """
        egap = '$\\varepsilon$$_{gap}$ ($\\varepsilon$$_{LUMO}$-$\\varepsilon$$_{HOMO}$)'
        voltage = 'Open-circuit voltage (V$_{OC}$)'
        assert 'Mass'  in prediction, "Mass prediction missing (check ML logic)"
        assert egap    in prediction, "E_gap prediction missing (check ML logic)"
        assert voltage in prediction, "V_OC prediction missing (check ML logic)"

        assert _almost_equal(prediction['Mass'][0], 250,  60.0), "Mass mean prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction['Mass'][1], 30.0, 40.0), "Mass sigma prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[egap][0], 2.6,  0.7), "E_gap mean prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[egap][1], 0.50, 0.55), "E_gap sigma prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[voltage][0], 1.0, 0.9), "V_OC mean prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[voltage][1], 0.8, 0.9), "V_OC sigma prediction beyond tolerance (check ML logic)"

    def test_predict(self):
        """
        Test predictions on the standard organic model

        This model is trained on HCEP data.  The prediction mirrors that on the
        organics demo script
        """

        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        inputs = [{"SMILES": "c1(C=O)cc(OC)c(O)cc1"}, ]
        vid = "177"

        resp = client.predict(vid, inputs, method="scalar")
        prediction = resp['candidates'][0]
        self._test_prediction_values(prediction)

    def test_predict_from_distribution(self):
        """
        Test predictions on the standard organic model

        Same as `test_predict` but using the `from_distribution` method
        """

        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        inputs = [{"SMILES": "c1(C=O)cc(OC)c(O)cc1"}, ]
        vid = "177"

        resp = client.predict(vid, inputs, method="from_distribution")
        prediction = resp['candidates'][0]
        self._test_prediction_values(prediction)

    def test_predict_custom(self):
        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        input = {"canary_x": "0.5", "temperature": "100", "canary_y": "0.75"}
        resp = client.predict_custom("canary", input)
        prediction = resp['candidates'][0]
        assert 'canary_zz' in prediction.keys()
        assert 'canary_z' in prediction.keys()

    def test_tsne(self):
        """
        Test that we can grab the t-SNE from a pre-trained view
        """
        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        resp = client.tsne("1623")

        assert len(resp) == 1, "Expected a single tSNE block but got {}".format(len(resp))

        tsne_y = resp[list(resp.keys())[0]]
        assert "x" in tsne_y, "Couldn't find x component of tsne projection"
        assert "y" in tsne_y, "Couldn't find y component of tsne projection"
        assert "z" in tsne_y, "Couldn't find property label for tsne projection"
        assert "uid" in tsne_y, "Couldn't find uid in tsne projection"
        assert "label" in tsne_y, "Couldn't find label in tsne projection"

        assert len(tsne_y["x"]) == len(tsne_y["y"]),     "tSNE components x and y had different lengths"
        assert len(tsne_y["x"]) == len(tsne_y["z"]),     "tSNE components x and z had different lengths"
        assert len(tsne_y["x"]) == len(tsne_y["label"]), "tSNE components x and uid had different lengths"
        assert len(tsne_y["x"]) == len(tsne_y["uid"]),   "tSNE components x and label had different lengths"

<commit_msg>add simple tests for search<commit_after>from citrination_client import CitrinationClient
from os import environ
import os
from json import loads
from pypif.obj.system import System
from pypif.pif import dump
from citrination_client.search import *
import random
import string

def _almost_equal(test_value, reference_value, tolerance=1.0e-9):
    """
    Numerical equality with a tolerance
    """
    return abs(test_value - reference_value) < tolerance

class TestClient():

    @classmethod
    def setup_class(self):
        self.client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        # Append dataset name with random string because one user can't have more than
        # one dataset with the same name
        random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
        dataset_name = "Tutorial dataset " + random_string
        self.set_id = loads(self.client.create_data_set(name=dataset_name, description="Dataset for tutorial", share=0).content.decode('utf-8'))['id']
        self.test_file_root = './citrination_client/tests/test_files/'

    def get_test_file_hierarchy_count(self):
        test_dir = self.test_file_root
        return sum([len(files) for r, d, files in os.walk(test_dir)])

    def test_start_client(self):
        assert self.client is not None

    def test_upload_pif(self):
        pif = System()
        pif.id = 0

        with open("tmp.json", "w") as fp:
            dump(pif, fp)
        response = loads(self.client.upload_file("tmp.json", self.set_id))
        assert response["message"] == "Upload is complete."

    def test_file_listing(self):
        src_path = self.test_file_root + "keys_and_values.json"
        dest_path = "test_file_list.json"
        self.client.upload(self.set_id, src_path, dest_path)
        listed_files = self.client.list_files(self.set_id, dest_path)["files"]
        assert len(listed_files) == 1
        assert listed_files[0] == dest_path

    def test_upload_directory(self):
        count_to_add = self.get_test_file_hierarchy_count()
        src_path = self.test_file_root
        dest_path = "test_directory_upload/"
        before_count = self.client.matched_file_count(self.set_id)
        self.client.upload(self.set_id, src_path, dest_path)
        after_count = self.client.matched_file_count(self.set_id)
        assert after_count == (before_count + count_to_add)

    def test_pif_search(self):
        response = self.client.pif_search(PifSystemReturningQuery(
            size=0,
            query=DataQuery(
                system=PifSystemQuery(
                    chemical_formula=ChemicalFieldQuery(
                        filter=ChemicalFilter(
                            equal='C22H15NSSi'))))))
        assert 5 == response.total_num_hits

    def test_pif_multi_search(self):
        response = self.client.pif_multi_search(MultiQuery(
            queries=[
                PifSystemReturningQuery(
                    size=0,
                    query=DataQuery(
                        system=PifSystemQuery(
                            chemical_formula=ChemicalFieldQuery(
                                filter=ChemicalFilter(
                                    equal='C22H15NSSi'))))),
                PifSystemReturningQuery(
                    size=0,
                    query=DataQuery(
                        system=PifSystemQuery(
                            chemical_formula=ChemicalFieldQuery(
                                filter=ChemicalFilter(
                                    equal='C22H12N4S3SeSi')))))
            ]))
        assert 2 == len(response.results)
        assert 5 == response.results[0].result.total_num_hits
        assert 1 == response.results[1].result.total_num_hits

    def test_dataset_search(self):
        response = self.client.dataset_search(DatasetReturningQuery(
            size=0,
            query=DataQuery(
                dataset=DatasetQuery(
                    id=Filter(equal='151278')))))
        assert 1 == response.total_num_hits

    @staticmethod
    def _test_prediction_values(prediction):
        """
        Assertions for the test_predict and test_predict_distribution methods
        """
        egap = '$\\varepsilon$$_{gap}$ ($\\varepsilon$$_{LUMO}$-$\\varepsilon$$_{HOMO}$)'
        voltage = 'Open-circuit voltage (V$_{OC}$)'
        assert 'Mass'  in prediction, "Mass prediction missing (check ML logic)"
        assert egap    in prediction, "E_gap prediction missing (check ML logic)"
        assert voltage in prediction, "V_OC prediction missing (check ML logic)"

        assert _almost_equal(prediction['Mass'][0], 250,  60.0), "Mass mean prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction['Mass'][1], 30.0, 40.0), "Mass sigma prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[egap][0], 2.6,  0.7), "E_gap mean prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[egap][1], 0.50, 0.55), "E_gap sigma prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[voltage][0], 1.0, 0.9), "V_OC mean prediction beyond tolerance (check ML logic)"
        assert _almost_equal(prediction[voltage][1], 0.8, 0.9), "V_OC sigma prediction beyond tolerance (check ML logic)"

    def test_predict(self):
        """
        Test predictions on the standard organic model

        This model is trained on HCEP data.  The prediction mirrors that on the
        organics demo script
        """

        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        inputs = [{"SMILES": "c1(C=O)cc(OC)c(O)cc1"}, ]
        vid = "177"

        resp = client.predict(vid, inputs, method="scalar")
        prediction = resp['candidates'][0]
        self._test_prediction_values(prediction)

    def test_predict_from_distribution(self):
        """
        Test predictions on the standard organic model

        Same as `test_predict` but using the `from_distribution` method
        """

        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        inputs = [{"SMILES": "c1(C=O)cc(OC)c(O)cc1"}, ]
        vid = "177"

        resp = client.predict(vid, inputs, method="from_distribution")
        prediction = resp['candidates'][0]
        self._test_prediction_values(prediction)

    def test_predict_custom(self):
        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        input = {"canary_x": "0.5", "temperature": "100", "canary_y": "0.75"}
        resp = client.predict_custom("canary", input)
        prediction = resp['candidates'][0]
        assert 'canary_zz' in prediction.keys()
        assert 'canary_z' in prediction.keys()

    def test_tsne(self):
        """
        Test that we can grab the t-SNE from a pre-trained view
        """
        client = CitrinationClient(environ['CITRINATION_API_KEY'], environ['CITRINATION_SITE'])
        resp = client.tsne("1623")

        assert len(resp) == 1, "Expected a single tSNE block but got {}".format(len(resp))

        tsne_y = resp[list(resp.keys())[0]]
        assert "x" in tsne_y, "Couldn't find x component of tsne projection"
        assert "y" in tsne_y, "Couldn't find y component of tsne projection"
        assert "z" in tsne_y, "Couldn't find property label for tsne projection"
        assert "uid" in tsne_y, "Couldn't find uid in tsne projection"
        assert "label" in tsne_y, "Couldn't find label in tsne projection"

        assert len(tsne_y["x"]) == len(tsne_y["y"]),     "tSNE components x and y had different lengths"
        assert len(tsne_y["x"]) == len(tsne_y["z"]),     "tSNE components x and z had different lengths"
        assert len(tsne_y["x"]) == len(tsne_y["label"]), "tSNE components x and uid had different lengths"
        assert len(tsne_y["x"]) == len(tsne_y["uid"]),   "tSNE components x and label had different lengths"

<|endoftext|>


<commit_before>"""
Tool for initial rpath fix for prebuilt binaries
"""
from __future__ import absolute_import, division, print_function

import os
import glob

from subprocess import CalledProcessError, check_output

# =============================================================================
if __name__ == '__main__':
  main_files = glob.glob('py2app/apptemplate/prebuilt/main*')
  secondary_files = glob.glob('py2app/apptemplate/prebuilt/secondary*')
  for bin_file in main_files + secondary_files:
    if os.path.isfile(bin_file):
      print(bin_file)
      libraries = list()
      try:
        libraries = check_output(['otool', '-L', bin_file]).decode('utf8').split('\n')
      except CalledProcessError:
        pass
      for line in libraries[1:]:
        lib = line.strip().split()
        if len(lib) > 0:
          lib = lib[0]
          new_lib = None
          if 'libgcc_s' in lib:
            new_lib = os.path.join('@rpath', 'lib', lib.split('/')[-1])
          if new_lib is not None:
            print('Changing {lib} to {new_lib}'.format(lib=lib, new_lib=new_lib))
            cmd = ['install_name_tool', '-change', lib, new_lib, bin_file]
            print(' '.join(cmd))
            output = check_output(cmd)
            cmd = ['install_name_tool', '-add_rpath', os.getenv('PREFIX'), bin_file]
            print(' '.join(cmd))
            output = check_output(cmd)
<commit_msg>Add bundletemplate files and try again<commit_after>"""
Tool for initial rpath fix for prebuilt binaries
"""
from __future__ import absolute_import, division, print_function

import os
import glob

from subprocess import CalledProcessError, check_output

# =============================================================================
if __name__ == '__main__':
  main_files = glob.glob('py2app/apptemplate/prebuilt/main*')
  secondary_files = glob.glob('py2app/apptemplate/prebuilt/secondary*')
  bundle_files = glob.glob('py2app/bundletemplate/prebuilt/main*')
  for bin_file in main_files + secondary_files + bundle_files:
    if os.path.isfile(bin_file):
      print(bin_file)
      libraries = list()
      try:
        libraries = check_output(['otool', '-L', bin_file]).decode('utf8').split('\n')
      except CalledProcessError:
        pass
      for line in libraries[1:]:
        lib = line.strip().split()
        if len(lib) > 0:
          lib = lib[0]
          new_lib = None
          if 'libgcc_s' in lib:
            new_lib = os.path.join('@rpath', lib.split('/')[-1])
          if new_lib is not None:
            print('Changing {lib} to {new_lib}'.format(lib=lib, new_lib=new_lib))
            cmd = ['install_name_tool', '-change', lib, new_lib, bin_file]
            print(' '.join(cmd))
            output = check_output(cmd)
            cmd = ['install_name_tool', '-add_rpath',
                   os.path.join('@loader_path', '..', 'lib'),
                   bin_file]
            print(' '.join(cmd))
            output = check_output(cmd)
<|endoftext|>


<commit_before>import os

import wx
from pyPdf import PdfFileWriter, PdfFileReader

# This class actually does the verification of splitting parameters and the
# writing of the new PDF files.

class WorkerWindow(wx.Frame):
    """
    WorkerWindow: the window that lets the user know what work is being done
                  and does all the pdf splitting and so forth.

    -----------------------------------------
    |                                       |
    | spce | arrow | Text | check/x | space |
    | spce | arrow | Text | check/x | space |
    | spce | arrow | Text | check/x | space |
    | spce | arrow | Text | check/x | space |
    |                                       |
    |              |DONE|                   |
    -----------------------------------------
    """

    def __init__(self, parent):
        # Initially passing off the PdfFileReader object to this class
        # self.le_pdf = pdf
        # Followed by setting up the UI
        self.parent = parent
        wx.Frame.__init__(self, parent=wx.GetApp().TopWindow, id=wx.ID_ANY,
            style=wx.RESIZE_BORDER|wx.CAPTION|wx.CLIP_CHILDREN|
            wx.FRAME_NO_TASKBAR|wx.FRAME_FLOAT_ON_PARENT,
            size=(300,110)
        )
        self.panel = wx.Panel(self)

        # Some basic time savers
        self.rows = parent.splitters
        self.le_splitters = []

        # Layout
        self.vbox = wx.BoxSizer(wx.VERTICAL)
        self.vbox.Add((0,10))

        # There will be several processes happening:

        # 1) Analyze input
        self.hb_analyze = wx.BoxSizer(wx.HORIZONTAL)
        self.hb_analyze.Add(wx.StaticText(self.panel, label=unichr(ord(u'\u27A4')), style=wx.ALIGN_RIGHT), proportion=1)
        self.hb_analyze.Add(wx.StaticText(self.panel, label="Analyzing input...", style=wx.ALIGN_LEFT), proportion=3)
        self.hb_analyze.Add(wx.StaticText(self.panel, label="", style=wx.ALIGN_LEFT), proportion=3)
        self.vbox.Add(self.hb_analyze, proportion=0, flag=wx.GROW)
        self.hb_analyze.ShowItems(False)

        # 2) Verify input
        self.hb_verify = wx.BoxSizer(wx.HORIZONTAL)
        self.hb_verify.Add(wx.StaticText(self.panel, label=unichr(ord(u'\u27A4')), style=wx.ALIGN_RIGHT), proportion=1)
        self.hb_verify.Add(wx.StaticText(self.panel, label="Verifying input...", style=wx.ALIGN_LEFT), proportion=3)
        self.hb_verify.Add(wx.StaticText(self.panel, label="", style=wx.ALIGN_LEFT), proportion=3)
        self.vbox.Add(self.hb_verify, proportion=0, flag=wx.GROW)
        self.hb_verify.ShowItems(False)

        """
        for new_row in range(0, self.rows):
            hbox = wx.BoxSizer(wx.HORIZONTAL)
            hbox.Add(wx.StaticText(self.panel, label=unichr(ord(u'\u27A4')), style=wx.ALIGN_RIGHT), proportion=1)
            hbox.Add(wx.StaticText(self.panel,
                    label="Generating " + parent.get_splitter(new_row).cmb_prefix.GetValue() + " documents...",
                    style=wx.ALIGN_LEFT),
                proportion=3
            )
            hbox.Add(wx.StaticText(self.panel, label="", style=wx.ALIGN_LEFT), proportion=3)
            self.vbox.Add(hbox, proportion=0, flag=wx.GROW)
            hbox.ShowItems(False)
            size = self.GetSize()
            self.SetSize((size.x, size.y+10))
        """
        #self.vbox.Add(wx.Button(self.panel, label="Close"), proportion=0, flag=wx.GROW)
        self.SetSize((self.GetSize().x, self.GetSize().y+(10*self.rows)))

        self.panel.SetSizer(self.vbox)
        self.panel.Layout()

    def process(self):
        """
        Analyze
        Verify
        Generate
        Done
        """

        # Begin by analyzing (or creating a list of splitpanels to work with
        self.hb_analyze.ShowItems(True)
        # Redraw
        self.panel.Layout()

        if not self.analyze_input():
            self.Destroy()
            return (False, "Failed to Analyze")
        else:
            # Ugh, magic numbers...
            # 2: "Success!"
            self.helper_gray(self.hb_analyze)
            self.hb_analyze.GetItem(2).GetWindow().SetLabel("Success!")
            self.hb_verify.ShowItems(True)

        # Redraw
        self.panel.Layout()

        if not self.verify_input():
            self.Destroy()
            return (False, "Failed to Verify")

    def analyze_input(self):
        """
        Generate a list of the splitpanels for ease of use
        """
        for k,v in enumerate(self.parent.shell_grid.GetChildren()):
            if k >= self.parent.SPLITTER_START_POS and k < len(self.parent.shell_grid.GetChildren())-1:
                window = v.GetWindow()
                # Check for whites
                if (window.ent_path.GetValue() == "" or
                    window.ent_split_rules.GetValue() == ""):
                    # Temp hack, because I can.
                    self.hb_analyze.GetItem(2).GetWindow().SetLabel("FAILED!")
                    self.hb_analyze.GetItem(2).GetWindow().SetForegroundColour("Red")
                    self.panel.Layout()
                    wx.MessageBox("Please fill out all fields before splitting.",
                        "info", wx.OK|wx.ICON_INFORMATION)
                    return False
                self.le_splitters.append(window)
        return True

    def verify_input(self):
        # Should always have analyzed before verify
        if not self.le_splitters:
            wx.MessageBox("Analyze should be ran first!", "Error", wx.OK|wx.ICON_ERROR)
            return False

        for section in self.le_splitters:
            # Verify output path
            if not os.path.isdir(os.path.join(section.ent_path.GetValue())):
                return False

            if not section.ent_split_rules.GetBackgroundColour() <> (255,192,203):
                return False

        return True

    def helper_gray(self, sizer):
        for child in sizer.GetChildren():
            child.GetWindow().SetForegroundColour('Gray')

if __name__ == '__main__':
    print "Not a standalone module"
    exit(0)
<commit_msg>Swapped the verify and analyze names. They seemed reversed, imo.<commit_after>import os

import wx
from pyPdf import PdfFileWriter, PdfFileReader

# This class actually does the verification of splitting parameters and the
# writing of the new PDF files.

class WorkerWindow(wx.Frame):
    """
    WorkerWindow: the window that lets the user know what work is being done
                  and does all the pdf splitting and so forth.

    -----------------------------------------
    |                                       |
    | spce | arrow | Text | check/x | space |
    | spce | arrow | Text | check/x | space |
    | spce | arrow | Text | check/x | space |
    | spce | arrow | Text | check/x | space |
    |                                       |
    |              |DONE|                   |
    -----------------------------------------
    """

    def __init__(self, parent):
        # Initially passing off the PdfFileReader object to this class
        # self.le_pdf = pdf
        # Followed by setting up the UI
        self.parent = parent
        wx.Frame.__init__(self, parent=wx.GetApp().TopWindow, id=wx.ID_ANY,
            style=wx.RESIZE_BORDER|wx.CAPTION|wx.CLIP_CHILDREN|
            wx.FRAME_NO_TASKBAR|wx.FRAME_FLOAT_ON_PARENT,
            size=(300,110)
        )
        self.panel = wx.Panel(self)

        # Some basic time savers
        self.rows = parent.splitters
        self.le_splitters = []

        # Layout
        self.vbox = wx.BoxSizer(wx.VERTICAL)
        self.vbox.Add((0,10))

        # 1) Verify input
        self.hb_verify = wx.BoxSizer(wx.HORIZONTAL)
        self.hb_verify.Add(wx.StaticText(self.panel, label=unichr(ord(u'\u27A4')), style=wx.ALIGN_RIGHT), proportion=1)
        self.hb_verify.Add(wx.StaticText(self.panel, label="Verifying input...", style=wx.ALIGN_LEFT), proportion=3)
        self.hb_verify.Add(wx.StaticText(self.panel, label="", style=wx.ALIGN_LEFT), proportion=3)
        self.vbox.Add(self.hb_verify, proportion=0, flag=wx.GROW)
        self.hb_verify.ShowItems(False)

        # 2) Analyze input
        self.hb_analyze = wx.BoxSizer(wx.HORIZONTAL)
        self.hb_analyze.Add(wx.StaticText(self.panel, label=unichr(ord(u'\u27A4')), style=wx.ALIGN_RIGHT), proportion=1)
        self.hb_analyze.Add(wx.StaticText(self.panel, label="Analyzing input...", style=wx.ALIGN_LEFT), proportion=3)
        self.hb_analyze.Add(wx.StaticText(self.panel, label="", style=wx.ALIGN_LEFT), proportion=3)
        self.vbox.Add(self.hb_analyze, proportion=0, flag=wx.GROW)
        self.hb_analyze.ShowItems(False)

        """
        for new_row in range(0, self.rows):
            hbox = wx.BoxSizer(wx.HORIZONTAL)
            hbox.Add(wx.StaticText(self.panel, label=unichr(ord(u'\u27A4')), style=wx.ALIGN_RIGHT), proportion=1)
            hbox.Add(wx.StaticText(self.panel,
                    label="Generating " + parent.get_splitter(new_row).cmb_prefix.GetValue() + " documents...",
                    style=wx.ALIGN_LEFT),
                proportion=3
            )
            hbox.Add(wx.StaticText(self.panel, label="", style=wx.ALIGN_LEFT), proportion=3)
            self.vbox.Add(hbox, proportion=0, flag=wx.GROW)
            hbox.ShowItems(False)
            size = self.GetSize()
            self.SetSize((size.x, size.y+10))
        """
        #self.vbox.Add(wx.Button(self.panel, label="Close"), proportion=0, flag=wx.GROW)
        self.SetSize((self.GetSize().x, self.GetSize().y+(10*self.rows)))

        self.panel.SetSizer(self.vbox)
        self.panel.Layout()

    def process(self):
        """
        Verify
        Analyze
        Generate
        Done
        """

        # Begin by analyzing (or creating a list of splitpanels to work with
        self.hb_verify.ShowItems(True)
        # Redraw
        self.panel.Layout()

        if not self.verify_input():
            self.Destroy()
            return (False, "Failed to Verify")
        else:
            self.helper_gray(self.hb_verify)
            self.hb_verify.GetItem(2).GetWindow().SetLabel("Success!")

        # Redraw
        self.hb_analyze.ShowItems(True)
        self.panel.Layout()

        if not self.analyze_input():
            self.Destroy()
            return (False, "Failed to Analyze")
        else:
            # Ugh, magic numbers...
            # 2: "Success!"
            self.helper_gray(self.hb_analyze)
            self.hb_analyze.GetItem(2).GetWindow().SetLabel("Success!")

        # Redraw
        self.panel.Layout()

    def verify_input(self):
        """
        Generate a list of the splitpanels for ease of use
        """
        for k,v in enumerate(self.parent.shell_grid.GetChildren()):
            if k >= self.parent.SPLITTER_START_POS and k < len(self.parent.shell_grid.GetChildren())-1:
                window = v.GetWindow()
                # Check for whites
                if (window.ent_path.GetValue() == "" or
                    window.ent_split_rules.GetValue() == ""):
                    # Temp hack, because I can.
                    self.hb_verify.GetItem(2).GetWindow().SetLabel("FAILED!")
                    self.hb_verify.GetItem(2).GetWindow().SetForegroundColour("Red")
                    self.panel.Layout()
                    wx.MessageBox("Please fill out all fields before splitting.",
                        "info", wx.OK|wx.ICON_INFORMATION)
                    return False
                self.le_splitters.append(window)
        return True

    def analyze_input(self):
        # Should always have analyzed before verify
        if not self.le_splitters:
            wx.MessageBox("Verify should be ran first!", "Error", wx.OK|wx.ICON_ERROR)
            return False

        for section in self.le_splitters:
            # Verify output path
            if not os.path.isdir(os.path.join(section.ent_path.GetValue())):
                return False

            if not section.ent_split_rules.GetBackgroundColour() <> (255,192,203):
                return False

        return True

    def helper_gray(self, sizer):
        for child in sizer.GetChildren():
            child.GetWindow().SetForegroundColour('Gray')

if __name__ == '__main__':
    print "Not a standalone module"
    exit(0)
<|endoftext|>


<commit_before>import re

from django.db import models
from django.core.exceptions import ValidationError

from cms.models import CMSPlugin
from filer.fields.file import FilerFileField
from filer.fields.image import FilerImageField
from taggit.managers import TaggableManager
import pubmed_lookup

class Publication(models.Model):

    pmid = models.IntegerField('PubMed ID',
        blank=True,
        help_text="Enter publication's PubMed ID.",
        unique=True,
    )
    pubmed_url = models.URLField('PubMed URL',
        blank=True,
        help_text="Enter publication's PubMed URL.",
    )
    redo_query = models.BooleanField('redo PubMed query?',
        default=False,
        help_text='Check this box to redo the PubMed query.
Any manual ' \
                  'changes to the PubMed metadata will be overwritten.',
    )
    no_query = models.BooleanField("Don't query PubMed",
        default=False,
        help_text="Check this box to prevent a PubMed query.
" \
                  "Instead, enter publication info manually in the section below labeled " \
                  "'Auto-generated PubMed Metadata'.
" \
                  "This option is useful for when there is no PubMed record for the publication.",
    )

    pdf = FilerFileField(
        blank=True,
        null=True,
        help_text="Upload/select a PDF for this publication.
" \
                  "Recommended naming format: '[mini citation]'.",
        related_name='%(app_label)s_%(class)s_pdf',
    )
    supplemental_pdf = FilerFileField(
        blank=True,
        null=True,
        help_text="Upload/select a supplemental PDF for this publication.
" \
                  "Recommended naming format: '[mini citation] - Supplement'.",
        related_name='%(app_label)s_%(class)s_supplemental_pdf',
    )

    image = FilerImageField(
        blank=True,
        null=True,
        help_text="Upload/select a representative image or figure " \
                  "for this publication.
" \
                  "Recommended naming format: '[mini citation] - Figure X'.",
        related_name='%(app_label)s_%(class)s_image',
    )

    tags = TaggableManager(
        help_text='Add keyword tags that represent this publication.',
    )

    title = models.CharField('title',
        blank=True,
        max_length=255,
    )
    authors = models.TextField('authors',
        blank=True,
    )
    first_author = models.CharField('first author',
        blank=True,
        max_length=255,
    )
    last_author = models.CharField('last author',
        blank=True,
        max_length=255,
    )
    journal = models.CharField('journal',
        blank=True,
        max_length=255,
    )
    year = models.CharField('year',
        blank=True,
        max_length=4,
    )
    month = models.CharField('month',
        blank=True,
        max_length=2,
    )
    day = models.CharField('day',
        blank=True,
        max_length=2,
    )
    url = models.URLField('publication URL',
        blank=True,
    )
    citation = models.TextField('citation',
        blank=True,
    )
    mini_citation = models.CharField('mini citation',
        blank=True,
        help_text='This field is auto-generated when a PubMed query ' \
                  'is made.
' \
                  'It is recommended to use this text when adding custom ' \
                  'names for uploaded files. See examples below.',
        max_length=255,
    )
    abstract = models.TextField('abstract',
        blank=True,
    )

    def clean(self):
        if not self.pk:
            if self.pmid and self.pubmed_url:
                raise ValidationError(
                    "Enter a PubMed ID or a PubMed URL, but not both.")

            if self.pmid == "" and self.pubmed_url == "":
                raise ValidationError("Enter a PubMed ID or a PubMed URL.")

    def save(self, *args, **kwargs):
        """
        Before saving, get publication's PubMed metadata if publication
        is not already in database or if 'redo_query' is True.
        """
        if self.no_query:
            if not self.pk or self.pmid > 0:
                pmid_min = Publication.objects.all().aggregate(
                    models.Min('pmid'))['pmid__min'] - 1
                self.pmid = min(0, pmid_min)

            self.pubmed_url = ''
            self.mini_citation = '{} - {} - {}'.format(
                self.first_author, self.year, self.journal)

        elif self.redo_query or not self.pk:
            if self.pmid:
                query = self.pmid
            else:
                query = self.pubmed_url

            email = ""    # FIX THIS: Use logged-in user's email
            lookup = pubmed_lookup.PubMedLookup(query, email)
            publication = pubmed_lookup.Publication(lookup)

            self.pmid = publication.pmid
            self.pubmed_url = publication.pubmed_url

            self.title = publication.title
            self.authors = publication.authors
            self.first_author = publication.first_author
            self.last_author = publication.last_author
            self.journal = publication.journal
            self.year = publication.year
            self.month = publication.month
            self.day = publication.day
            self.url = publication.url
            self.citation = publication.cite()
            self.mini_citation = publication.cite_mini()
            self.abstract = publication.abstract

        self.redo_query = False

        super().save(*args, **kwargs)

    def __str__(self):
        return "{} - {} - {} - {} [{}]".format(self.year,
            self.first_author, self.journal, self.title, str(self.pmid),)

    class Meta:
        ordering = ('-year', '-month', '-day', 'first_author')

class PublicationSet(models.Model):

    name = models.CharField('name',
        help_text="Enter a unique name for this Publication Set.
" \
                  "This won't be displayed on the site.",
        max_length=255,
        unique=True,
    )
    label = models.CharField('label',
        default='Publications',
        help_text='Enter a label for this Publication Set.
' \
                  'This will be the heading displayed above the publications.',
        max_length=255,
    )
    description = models.TextField('description',
        blank=True,
        help_text='Enter a description of this Publication Set.',
    )

    pagination = models.PositiveIntegerField('pubs per page',
        default=0,
        help_text="How many publications should be displayed per page?
" \
                  "To show all at once, enter '0'.",
    )

    bulk_pubmed_query = models.TextField('Bulk Query',
        blank=True,
        help_text='Enter PubMed IDs and/or PubMed URLs to get or create ' \
                  'multiple Publications and add them to this Publication Set.
' \
                  'PubMed IDs/URLs must be separated by commas or whitespace.
' \
                  'To add files and tags to publication records, create publications ' \
                  'individually via the Publication Admin (or below).',
    )
    publications = models.ManyToManyField(Publication,
        blank=True,
        null=True,
    )

    searchable = models.BooleanField('searchable?',
        default=True,
        help_text='Enable publication search and keyword filter.',
    )

    tags = TaggableManager(
        help_text='Add keyword tags that represent this publication set.',
    )

    def perform_bulk_pubmed_query(self):
        """
        If 'bulk_pubmed_query' contains any content, perform a bulk PubMed query,
        add the publications to the publication set, and save.
        """
        if self.bulk_pubmed_query:
            failed_queries = []
            pmid_list = re.findall(r'(\d+)(?:[\s,]+|$)', self.bulk_pubmed_query)

            for pmid in pmid_list:
                try:
                    p, created = Publication.objects.get_or_create(pmid=pmid)
                except:
                    failed_queries.append(pmid)
                else:
                    self.publications.add(p.id)

            if failed_queries:
                failed_queries.sort(key=int)
                self.bulk_pubmed_query = 'FAILED QUERIES: {}'.format(', '.join(failed_queries))
            else:
                self.bulk_pubmed_query = ''

    def clean(self):
        """
        Require creation of Publication Set before performing a Bulk PubMed Query.
        """
        if self.bulk_pubmed_query and self.pk is None:
            raise ValidationError(
                'Can only perform a Bulk PubMed Query with an existing Publication Set. ' \
                'First create this Publication Set and then do the Bulk PubMed Query.'
            )

    def save(self, *args, **kwargs):
        """
        Before saving, execute 'perform_bulk_pubmed_query()'.
        """
        self.perform_bulk_pubmed_query()
        super().save(*args, **kwargs)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ('name',)

class PublicationSetPlugin(CMSPlugin):
    publication_set = models.ForeignKey('cms_lab_publications.PublicationSet')

    def __str__(self):
        return self.publication_set.name
<commit_msg>Allow 'tags' fields to be blank<commit_after>import re

from django.db import models
from django.core.exceptions import ValidationError

from cms.models import CMSPlugin
from filer.fields.file import FilerFileField
from filer.fields.image import FilerImageField
from taggit.managers import TaggableManager
import pubmed_lookup

class Publication(models.Model):

    pmid = models.IntegerField('PubMed ID',
        blank=True,
        help_text="Enter publication's PubMed ID.",
        unique=True,
    )
    pubmed_url = models.URLField('PubMed URL',
        blank=True,
        help_text="Enter publication's PubMed URL.",
    )
    redo_query = models.BooleanField('redo PubMed query?',
        default=False,
        help_text='Check this box to redo the PubMed query.
Any manual ' \
                  'changes to the PubMed metadata will be overwritten.',
    )
    no_query = models.BooleanField("Don't query PubMed",
        default=False,
        help_text="Check this box to prevent a PubMed query.
" \
                  "Instead, enter publication info manually in the section below labeled " \
                  "'Auto-generated PubMed Metadata'.
" \
                  "This option is useful for when there is no PubMed record for the publication.",
    )

    pdf = FilerFileField(
        blank=True,
        null=True,
        help_text="Upload/select a PDF for this publication.
" \
                  "Recommended naming format: '[mini citation]'.",
        related_name='%(app_label)s_%(class)s_pdf',
    )
    supplemental_pdf = FilerFileField(
        blank=True,
        null=True,
        help_text="Upload/select a supplemental PDF for this publication.
" \
                  "Recommended naming format: '[mini citation] - Supplement'.",
        related_name='%(app_label)s_%(class)s_supplemental_pdf',
    )

    image = FilerImageField(
        blank=True,
        null=True,
        help_text="Upload/select a representative image or figure " \
                  "for this publication.
" \
                  "Recommended naming format: '[mini citation] - Figure X'.",
        related_name='%(app_label)s_%(class)s_image',
    )

    tags = TaggableManager(
        help_text='Add keyword tags that represent this publication.',
        blank=True,
    )

    title = models.CharField('title',
        blank=True,
        max_length=255,
    )
    authors = models.TextField('authors',
        blank=True,
    )
    first_author = models.CharField('first author',
        blank=True,
        max_length=255,
    )
    last_author = models.CharField('last author',
        blank=True,
        max_length=255,
    )
    journal = models.CharField('journal',
        blank=True,
        max_length=255,
    )
    year = models.CharField('year',
        blank=True,
        max_length=4,
    )
    month = models.CharField('month',
        blank=True,
        max_length=2,
    )
    day = models.CharField('day',
        blank=True,
        max_length=2,
    )
    url = models.URLField('publication URL',
        blank=True,
    )
    citation = models.TextField('citation',
        blank=True,
    )
    mini_citation = models.CharField('mini citation',
        blank=True,
        help_text='This field is auto-generated when a PubMed query ' \
                  'is made.
' \
                  'It is recommended to use this text when adding custom ' \
                  'names for uploaded files. See examples below.',
        max_length=255,
    )
    abstract = models.TextField('abstract',
        blank=True,
    )

    def clean(self):
        if not self.pk:
            if self.pmid and self.pubmed_url:
                raise ValidationError(
                    "Enter a PubMed ID or a PubMed URL, but not both.")

            if self.pmid == "" and self.pubmed_url == "":
                raise ValidationError("Enter a PubMed ID or a PubMed URL.")

    def save(self, *args, **kwargs):
        """
        Before saving, get publication's PubMed metadata if publication
        is not already in database or if 'redo_query' is True.
        """
        if self.no_query:
            if not self.pk or self.pmid > 0:
                pmid_min = Publication.objects.all().aggregate(
                    models.Min('pmid'))['pmid__min'] - 1
                self.pmid = min(0, pmid_min)

            self.pubmed_url = ''
            self.mini_citation = '{} - {} - {}'.format(
                self.first_author, self.year, self.journal)

        elif self.redo_query or not self.pk:
            if self.pmid:
                query = self.pmid
            else:
                query = self.pubmed_url

            email = ""    # FIX THIS: Use logged-in user's email
            lookup = pubmed_lookup.PubMedLookup(query, email)
            publication = pubmed_lookup.Publication(lookup)

            self.pmid = publication.pmid
            self.pubmed_url = publication.pubmed_url

            self.title = publication.title
            self.authors = publication.authors
            self.first_author = publication.first_author
            self.last_author = publication.last_author
            self.journal = publication.journal
            self.year = publication.year
            self.month = publication.month
            self.day = publication.day
            self.url = publication.url
            self.citation = publication.cite()
            self.mini_citation = publication.cite_mini()
            self.abstract = publication.abstract

        self.redo_query = False

        super().save(*args, **kwargs)

    def __str__(self):
        return "{} - {} - {} - {} [{}]".format(self.year,
            self.first_author, self.journal, self.title, str(self.pmid),)

    class Meta:
        ordering = ('-year', '-month', '-day', 'first_author')

class PublicationSet(models.Model):

    name = models.CharField('name',
        help_text="Enter a unique name for this Publication Set.
" \
                  "This won't be displayed on the site.",
        max_length=255,
        unique=True,
    )
    label = models.CharField('label',
        default='Publications',
        help_text='Enter a label for this Publication Set.
' \
                  'This will be the heading displayed above the publications.',
        max_length=255,
    )
    description = models.TextField('description',
        blank=True,
        help_text='Enter a description of this Publication Set.',
    )

    pagination = models.PositiveIntegerField('pubs per page',
        default=0,
        help_text="How many publications should be displayed per page?
" \
                  "To show all at once, enter '0'.",
    )

    bulk_pubmed_query = models.TextField('Bulk Query',
        blank=True,
        help_text='Enter PubMed IDs and/or PubMed URLs to get or create ' \
                  'multiple Publications and add them to this Publication Set.
' \
                  'PubMed IDs/URLs must be separated by commas or whitespace.
' \
                  'To add files and tags to publication records, create publications ' \
                  'individually via the Publication Admin (or below).',
    )
    publications = models.ManyToManyField(Publication,
        blank=True,
        null=True,
    )

    searchable = models.BooleanField('searchable?',
        default=True,
        help_text='Enable publication search and keyword filter.',
    )

    tags = TaggableManager(
        help_text='Add keyword tags that represent this publication set.',
        blank=True,
    )

    def perform_bulk_pubmed_query(self):
        """
        If 'bulk_pubmed_query' contains any content, perform a bulk PubMed query,
        add the publications to the publication set, and save.
        """
        if self.bulk_pubmed_query:
            failed_queries = []
            pmid_list = re.findall(r'(\d+)(?:[\s,]+|$)', self.bulk_pubmed_query)

            for pmid in pmid_list:
                try:
                    p, created = Publication.objects.get_or_create(pmid=pmid)
                except:
                    failed_queries.append(pmid)
                else:
                    self.publications.add(p.id)

            if failed_queries:
                failed_queries.sort(key=int)
                self.bulk_pubmed_query = 'FAILED QUERIES: {}'.format(', '.join(failed_queries))
            else:
                self.bulk_pubmed_query = ''

    def clean(self):
        """
        Require creation of Publication Set before performing a Bulk PubMed Query.
        """
        if self.bulk_pubmed_query and self.pk is None:
            raise ValidationError(
                'Can only perform a Bulk PubMed Query with an existing Publication Set. ' \
                'First create this Publication Set and then do the Bulk PubMed Query.'
            )

    def save(self, *args, **kwargs):
        """
        Before saving, execute 'perform_bulk_pubmed_query()'.
        """
        self.perform_bulk_pubmed_query()
        super().save(*args, **kwargs)

    def __str__(self):
        return self.name

    class Meta:
        ordering = ('name',)

class PublicationSetPlugin(CMSPlugin):
    publication_set = models.ForeignKey('cms_lab_publications.PublicationSet')

    def __str__(self):
        return self.publication_set.name
<|endoftext|>


<commit_before>"""
Forever.fm Server
by @psobot, Nov 3 2012
"""

import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG,format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
import config
import apikeys
import database

import os
import sys
import json
import lame
import copy
import time
import info
import uuid
import random
import datetime
import threading
import traceback
import tornado.web
import tornado.ioloop
import tornado.template
import tornadio2.server
import multiprocessing
import pyechonest.config

from mixer import Mixer
from daemon import Daemon
from utils import daemonize
from listeners import Listeners
from assetcompiler import compiled
from sockethandler import SocketHandler
from bufferedqueue import BufferedReadQueue
from monitor import MonitorHandler, MonitorSocket, monitordaemon

#   API Key setup
pyechonest.config.ECHO_NEST_API_KEY = apikeys.ECHO_NEST_API_KEY

started_at_timestamp = time.time()
started_at = datetime.datetime.utcnow()

test = 'test' in sys.argv
SECONDS_PER_FRAME = lame.SAMPLES_PER_FRAME / 44100.0

templates = tornado.template.Loader(config.template_dir)
templates.autoescape = None
first_frame = threading.Semaphore(0)

class MainHandler(tornado.web.RequestHandler):
	mtime = 0
	template = 'index.html'

	def __gen(self):
		debug = self.get_argument('__debug', None)
		if debug is None:
			debug = self.request.host.startswith("localhost")
		else:
			debug = debug in ["True", 1, "on"]
		kwargs = {
			'debug': debug,
			'compiled': compiled,
			'open': len(StreamHandler.relays) > 0 or debug,
			'endpoint': StreamHandler.relay_url() if not debug else "/all.mp3",
			'complete_length': datetime.timedelta(seconds=int(database.get_complete_length()))
		}
		if os.path.getmtime(config.template_dir + self.template) > self.mtime:
			templates.reset()
			self.mtime = time.time()
		return templates.load(self.template).generate(**kwargs)

	def head(self):
		self.__gen()
		self.finish()

	def get(self):
		self.finish(self.__gen())

class InfoHandler(tornado.web.RequestHandler):
	actions = []
	started = None
	samples = 0L
	duration = 0.

	@classmethod
	def add(self, data):
		if not self.actions:
			self.started = time.time()
		self.samples += data['samples']
		self.duration += data['duration']
		self.clean()
		log.info("Adding track info. Currently holding info for %d tracks.",
				 len(self.actions))
		self.actions.append(data)
		SocketHandler.on_segment(data)

	@classmethod
	def clean(cls):
		now = time.time() - config.past_played_buffer
		while cls.actions and cls.actions[0]['time'] + cls.actions[0]['duration'] < now:
			cls.actions.pop(0)

	@classmethod
	def stats(cls):
		return {
			"started": cls.started,
			"samples": cls.samples,
			"duration": cls.duration,
		}

	def get(self):
		self.set_header("Content-Type", "application/json")
		try:
			now = self.get_argument('now', None)
			if now:
				now = time.time()
				for _action in self.actions:
					if _action['time'] < now \
					and _action['time'] + _action['duration'] > now:
						action = copy.copy(_action)
						self.write(json.dumps({'frame': action, 'now': now},
								   ensure_ascii=False).encode('utf-8'))
						return
				self.write(json.dumps([]))
			else:
				self.write(json.dumps(self.actions, ensure_ascii=False).encode('utf-8'))
		except Exception:
			log.error("Could not send info burst:\n%s", traceback.format_exc())
			log.error("Data:\n%s", self.actions)
			self.write(json.dumps([]))

class TimingHandler(tornado.web.RequestHandler):
	def get(self):
		self.set_header("Content-Type", "application/json")
		self.write(json.dumps({"time": time.time() * 1000}, ensure_ascii=False).encode('utf-8'))

class StreamHandler(tornado.web.RequestHandler):
	relays = []
	listeners = []

	@classmethod
	def relay_url(cls):
		if len(cls.relays) == 1:
			return cls.relays[0].url
		elif len(cls.relays) > 1:
			choices = [relay for relay in cls.relays for _ in xrange(0, relay.weight)]
			return random.choice(choices).url
		else:
			return ""

	@classmethod
	def stream_frames(cls):
		try:
			cls.relays.broadcast()
		except Exception:
			log.error("Could not broadcast due to: \n%s", traceback.format_exc())

	@classmethod
	def check(cls):
		#   TODO: This should do HTTP requests to ensure that all relays are
		#   still up
		pass

	def head(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua or len(self.relays) == 0:
				log.info("Got HEAD for relay at %s.", ip)
				self.set_header("Content-Type", "audio/mpeg")
				self.finish()
			else:
				if not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.head:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	@tornado.web.asynchronous
	def get(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua:
				url = self.request.headers['X-Relay-Addr']
				if not url.startswith('http://'):
					url = "http://" + url
				port = self.request.headers['X-Relay-Port']
				self.weight = int(self.request.headers.get('X-Relay-Weight', 1))
				log.info("Added new relay at %s:%s with weight %d.", url, port, self.weight)
				self.set_header("Content-Type", "audio/mpeg")
				self.url = "%s:%s/all.mp3" % (url, port)
				self.relays.append(self)
			else:
				if self.request.host.startswith("localhost:8193"):
					log.info("Added new debug listener at %s.", ip)
					self.set_header("Content-Type", "audio/mpeg")
					self.relays.append(self)
				elif not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.get:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	def on_finish(self):
		if self in self.relays:
			self.relays.remove(self)
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			log.info("Removed relay at %s with weight %d.", ip, getattr(self,"weight",0))

class SocketConnection(tornadio2.conn.SocketConnection):
	__endpoints__ = {
		"/info.websocket": SocketHandler,   #TODO: Rename
		"/monitor.websocket": MonitorSocket
	}

class Userform(tornado.web.RequestHandler):
	def get(self):
		kwargs = {}
		self.write(templates.load("fileuploadform.html").generate(**kwargs))

class Upload(tornado.web.RequestHandler):
	def post(self):
		fileinfo = self.request.files['filearg'][0]
		database.create_track(fileinfo['body'], fileinfo['filename'], self.request.arguments)
		self.finish("Thank you for your submission.")

def admin_page(deleted=0, updated=0):
	return templates.load("administration.html").generate(
		all_tracks=database.get_many_mp3(status="all", order_by='id'),
		deleted=deleted, updated=updated, compiled=compiled,
		delete_url=apikeys.delete_url, edit_url=apikeys.edit_url,
	)

class DeleteTrack(tornado.web.RequestHandler):
	def get(self, input):
		input = int(input) # TODO: If intification fails, send back a tidy error message, rather than just quietly deleting nothing
		log.info("Yo we got input: %r", input)
		database.delete_track(input)
		self.write(admin_page(deleted=input))

class EditTrack(tornado.web.RequestHandler):
	def get(self, input):
		log.info("Yo we got input: %r", str(input))
		self.write(templates.load("audition.html").generate(admin_url=apikeys.admin_url,
		track=database.get_single_track(int(input)), compiled=compiled))

class AdminRender(tornado.web.RequestHandler):
	def get(self):
		self.write(admin_page())

	def post(self):
		id=int(self.request.arguments['id'][0])
		database.update_track(id, self.request.arguments)
		self.write(admin_page(updated=id))

class TrackArtwork(tornado.web.RequestHandler):
	def get(self, id):
		art = database.get_track_artwork(int(id))
		if art is None:
			self.send_error(404)
		else:
			self.set_header("Content-Type","image/jpeg")
			self.write(str(art))

if __name__ == "__main__":
	Daemon()

	log.info("Starting %s...", config.app_name)

	track_queue = multiprocessing.Queue(1)
	log.info("Initializing read queue to hold %2.2f seconds of audio.",
			 config.frontend_buffer)
	v2_queue = BufferedReadQueue(int(config.frontend_buffer / SECONDS_PER_FRAME))
	info_queue = multiprocessing.Queue()

	mixer = Mixer(track_queue,v2_queue.raw,info_queue)
	mixer.start()

	daemonize(database.enqueue_tracks, track_queue)
	daemonize(info.generate, info_queue, first_frame, InfoHandler)
	StreamHandler.relays = Listeners(v2_queue, "All", first_frame)
	daemonize(monitordaemon,StreamHandler.relays,InfoHandler.stats,{"mp3_queue":v2_queue})

	tornado.ioloop.PeriodicCallback(InfoHandler.clean, 5 * 1000).start()
	tornado.ioloop.PeriodicCallback(StreamHandler.check, 10 * 1000).start()

	application = tornado.web.Application(
		tornadio2.TornadioRouter(SocketConnection).apply_routes([
			# Static assets for local development
			(r"/(favicon\.ico)", tornado.web.StaticFileHandler, {"path": "static/img/"}),
			(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": "static/"}),
			(r"/audio/(.*)", tornado.web.StaticFileHandler, {"path": "audio/"}),

			(r"/timing\.json", TimingHandler),

			(r"/all\.json", InfoHandler),
			(r"/all\.mp3", StreamHandler),

			(r"/monitor", MonitorHandler),
			(r"/", MainHandler),
			(r"/submit", Userform),
			(r"/upload", Upload),
			(apikeys.admin_url, AdminRender),
			(apikeys.delete_url+"/([0-9]+)", DeleteTrack),
			(apikeys.edit_url+"/([0-9]+)", EditTrack),
			(r"/artwork/([0-9]+).jpg", TrackArtwork),
		]),
		socket_io_port=config.socket_port,
		enabled_protocols=['websocket', 'xhr-multipart', 'xhr-polling', 'jsonp-polling']
	)

	frame_sender = tornado.ioloop.PeriodicCallback(
		StreamHandler.stream_frames, SECONDS_PER_FRAME * 1000
	)
	frame_sender.start()

	application.listen(config.http_port)
	try:
		tornadio2.server.SocketServer(application)
	except KeyboardInterrupt:
		mixer.terminate()
		raise
<commit_msg>compiled dependency added.<commit_after>"""
Forever.fm Server
by @psobot, Nov 3 2012
"""

import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG,format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
import config
import apikeys
import database

import os
import sys
import json
import lame
import copy
import time
import info
import uuid
import random
import datetime
import threading
import traceback
import tornado.web
import tornado.ioloop
import tornado.template
import tornadio2.server
import multiprocessing
import pyechonest.config

from mixer import Mixer
from daemon import Daemon
from utils import daemonize
from listeners import Listeners
from assetcompiler import compiled
from sockethandler import SocketHandler
from bufferedqueue import BufferedReadQueue
from monitor import MonitorHandler, MonitorSocket, monitordaemon

#   API Key setup
pyechonest.config.ECHO_NEST_API_KEY = apikeys.ECHO_NEST_API_KEY

started_at_timestamp = time.time()
started_at = datetime.datetime.utcnow()

test = 'test' in sys.argv
SECONDS_PER_FRAME = lame.SAMPLES_PER_FRAME / 44100.0

templates = tornado.template.Loader(config.template_dir)
templates.autoescape = None
first_frame = threading.Semaphore(0)

class MainHandler(tornado.web.RequestHandler):
	mtime = 0
	template = 'index.html'

	def __gen(self):
		debug = self.get_argument('__debug', None)
		if debug is None:
			debug = self.request.host.startswith("localhost")
		else:
			debug = debug in ["True", 1, "on"]
		kwargs = {
			'debug': debug,
			'compiled': compiled,
			'open': len(StreamHandler.relays) > 0 or debug,
			'endpoint': StreamHandler.relay_url() if not debug else "/all.mp3",
			'complete_length': datetime.timedelta(seconds=int(database.get_complete_length()))
		}
		if os.path.getmtime(config.template_dir + self.template) > self.mtime:
			templates.reset()
			self.mtime = time.time()
		return templates.load(self.template).generate(**kwargs)

	def head(self):
		self.__gen()
		self.finish()

	def get(self):
		self.finish(self.__gen())

class InfoHandler(tornado.web.RequestHandler):
	actions = []
	started = None
	samples = 0L
	duration = 0.

	@classmethod
	def add(self, data):
		if not self.actions:
			self.started = time.time()
		self.samples += data['samples']
		self.duration += data['duration']
		self.clean()
		log.info("Adding track info. Currently holding info for %d tracks.",
				 len(self.actions))
		self.actions.append(data)
		SocketHandler.on_segment(data)

	@classmethod
	def clean(cls):
		now = time.time() - config.past_played_buffer
		while cls.actions and cls.actions[0]['time'] + cls.actions[0]['duration'] < now:
			cls.actions.pop(0)

	@classmethod
	def stats(cls):
		return {
			"started": cls.started,
			"samples": cls.samples,
			"duration": cls.duration,
		}

	def get(self):
		self.set_header("Content-Type", "application/json")
		try:
			now = self.get_argument('now', None)
			if now:
				now = time.time()
				for _action in self.actions:
					if _action['time'] < now \
					and _action['time'] + _action['duration'] > now:
						action = copy.copy(_action)
						self.write(json.dumps({'frame': action, 'now': now},
								   ensure_ascii=False).encode('utf-8'))
						return
				self.write(json.dumps([]))
			else:
				self.write(json.dumps(self.actions, ensure_ascii=False).encode('utf-8'))
		except Exception:
			log.error("Could not send info burst:\n%s", traceback.format_exc())
			log.error("Data:\n%s", self.actions)
			self.write(json.dumps([]))

class TimingHandler(tornado.web.RequestHandler):
	def get(self):
		self.set_header("Content-Type", "application/json")
		self.write(json.dumps({"time": time.time() * 1000}, ensure_ascii=False).encode('utf-8'))

class StreamHandler(tornado.web.RequestHandler):
	relays = []
	listeners = []

	@classmethod
	def relay_url(cls):
		if len(cls.relays) == 1:
			return cls.relays[0].url
		elif len(cls.relays) > 1:
			choices = [relay for relay in cls.relays for _ in xrange(0, relay.weight)]
			return random.choice(choices).url
		else:
			return ""

	@classmethod
	def stream_frames(cls):
		try:
			cls.relays.broadcast()
		except Exception:
			log.error("Could not broadcast due to: \n%s", traceback.format_exc())

	@classmethod
	def check(cls):
		#   TODO: This should do HTTP requests to ensure that all relays are
		#   still up
		pass

	def head(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua or len(self.relays) == 0:
				log.info("Got HEAD for relay at %s.", ip)
				self.set_header("Content-Type", "audio/mpeg")
				self.finish()
			else:
				if not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.head:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	@tornado.web.asynchronous
	def get(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua:
				url = self.request.headers['X-Relay-Addr']
				if not url.startswith('http://'):
					url = "http://" + url
				port = self.request.headers['X-Relay-Port']
				self.weight = int(self.request.headers.get('X-Relay-Weight', 1))
				log.info("Added new relay at %s:%s with weight %d.", url, port, self.weight)
				self.set_header("Content-Type", "audio/mpeg")
				self.url = "%s:%s/all.mp3" % (url, port)
				self.relays.append(self)
			else:
				if self.request.host.startswith("localhost:8193"):
					log.info("Added new debug listener at %s.", ip)
					self.set_header("Content-Type", "audio/mpeg")
					self.relays.append(self)
				elif not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.get:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	def on_finish(self):
		if self in self.relays:
			self.relays.remove(self)
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			log.info("Removed relay at %s with weight %d.", ip, getattr(self,"weight",0))

class SocketConnection(tornadio2.conn.SocketConnection):
	__endpoints__ = {
		"/info.websocket": SocketHandler,   #TODO: Rename
		"/monitor.websocket": MonitorSocket
	}

class Userform(tornado.web.RequestHandler):
	def get(self):
		self.write(templates.load("fileuploadform.html").generate(compiled=compiled))

class Upload(tornado.web.RequestHandler):
	def post(self):
		fileinfo = self.request.files['filearg'][0]
		database.create_track(fileinfo['body'], fileinfo['filename'], self.request.arguments)
		self.finish("Thank you for your submission.")

def admin_page(deleted=0, updated=0):
	return templates.load("administration.html").generate(
		all_tracks=database.get_many_mp3(status="all", order_by='id'),
		deleted=deleted, updated=updated, compiled=compiled,
		delete_url=apikeys.delete_url, edit_url=apikeys.edit_url,
	)

class DeleteTrack(tornado.web.RequestHandler):
	def get(self, input):
		input = int(input) # TODO: If intification fails, send back a tidy error message, rather than just quietly deleting nothing
		log.info("Yo we got input: %r", input)
		database.delete_track(input)
		self.write(admin_page(deleted=input))

class EditTrack(tornado.web.RequestHandler):
	def get(self, input):
		log.info("Yo we got input: %r", str(input))
		self.write(templates.load("audition.html").generate(admin_url=apikeys.admin_url,
		track=database.get_single_track(int(input)), compiled=compiled))

class AdminRender(tornado.web.RequestHandler):
	def get(self):
		self.write(admin_page())

	def post(self):
		id=int(self.request.arguments['id'][0])
		database.update_track(id, self.request.arguments)
		self.write(admin_page(updated=id))

class TrackArtwork(tornado.web.RequestHandler):
	def get(self, id):
		art = database.get_track_artwork(int(id))
		if art is None:
			self.send_error(404)
		else:
			self.set_header("Content-Type","image/jpeg")
			self.write(str(art))

if __name__ == "__main__":
	Daemon()

	log.info("Starting %s...", config.app_name)

	track_queue = multiprocessing.Queue(1)
	log.info("Initializing read queue to hold %2.2f seconds of audio.",
			 config.frontend_buffer)
	v2_queue = BufferedReadQueue(int(config.frontend_buffer / SECONDS_PER_FRAME))
	info_queue = multiprocessing.Queue()

	mixer = Mixer(track_queue,v2_queue.raw,info_queue)
	mixer.start()

	daemonize(database.enqueue_tracks, track_queue)
	daemonize(info.generate, info_queue, first_frame, InfoHandler)
	StreamHandler.relays = Listeners(v2_queue, "All", first_frame)
	daemonize(monitordaemon,StreamHandler.relays,InfoHandler.stats,{"mp3_queue":v2_queue})

	tornado.ioloop.PeriodicCallback(InfoHandler.clean, 5 * 1000).start()
	tornado.ioloop.PeriodicCallback(StreamHandler.check, 10 * 1000).start()

	application = tornado.web.Application(
		tornadio2.TornadioRouter(SocketConnection).apply_routes([
			# Static assets for local development
			(r"/(favicon\.ico)", tornado.web.StaticFileHandler, {"path": "static/img/"}),
			(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": "static/"}),
			(r"/audio/(.*)", tornado.web.StaticFileHandler, {"path": "audio/"}),

			(r"/timing\.json", TimingHandler),

			(r"/all\.json", InfoHandler),
			(r"/all\.mp3", StreamHandler),

			(r"/monitor", MonitorHandler),
			(r"/", MainHandler),
			(r"/submit", Userform),
			(r"/upload", Upload),
			(apikeys.admin_url, AdminRender),
			(apikeys.delete_url+"/([0-9]+)", DeleteTrack),
			(apikeys.edit_url+"/([0-9]+)", EditTrack),
			(r"/artwork/([0-9]+).jpg", TrackArtwork),
		]),
		socket_io_port=config.socket_port,
		enabled_protocols=['websocket', 'xhr-multipart', 'xhr-polling', 'jsonp-polling']
	)

	frame_sender = tornado.ioloop.PeriodicCallback(
		StreamHandler.stream_frames, SECONDS_PER_FRAME * 1000
	)
	frame_sender.start()

	application.listen(config.http_port)
	try:
		tornadio2.server.SocketServer(application)
	except KeyboardInterrupt:
		mixer.terminate()
		raise
<|endoftext|>


<commit_before>from . import modules
from . import _utils

class Module(modules.Module):
    def before_setup(self, properties, arguments, beans):
        aws_access_key = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_access_key',
            log_message='[AWS] No aws_access_key found in properties'
        )
        aws_access_secret_key = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_access_secret_key',
            log_message='[AWS] No aws_access_secret_key found in properties'
        )
        aws_region = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_region',
            log_message='[AWS] No aws_region found in properties'
        )
        endpoint_url = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_endpoint_url',
            log_message='[AWS] No aws_endpoint_url found in properties'
        )

        beans['aws_credentials'] = {
            'aws_access_key_id': aws_access_key,
            'aws_secret_access_key': aws_access_secret_key,
            'region_name': aws_region,
            'endpoint_url': endpoint_url
        }

    def after_setup(self, properties, arguments, context, beans_initializer):
        pass

    @staticmethod
    def _filter_missing_endpoint(endpoint_url):
        return endpoint_url if len(endpoint_url) > 0 else None

def get_credentials(context):
    return context.aws_credentials
<commit_msg>removed redundant function<commit_after>from . import modules
from . import _utils

class Module(modules.Module):
    def before_setup(self, properties, arguments, beans):
        aws_access_key = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_access_key',
            log_message='[AWS] No aws_access_key found in properties'
        )
        aws_access_secret_key = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_access_secret_key',
            log_message='[AWS] No aws_access_secret_key found in properties'
        )
        aws_region = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_region',
            log_message='[AWS] No aws_region found in properties'
        )
        endpoint_url = _utils.resolve_property_or_report_error(
            properties=properties,
            key='aws_endpoint_url',
            log_message='[AWS] No aws_endpoint_url found in properties'
        )

        beans['aws_credentials'] = {
            'aws_access_key_id': aws_access_key,
            'aws_secret_access_key': aws_access_secret_key,
            'region_name': aws_region,
            'endpoint_url': endpoint_url
        }

    def after_setup(self, properties, arguments, context, beans_initializer):
        pass

def get_credentials(context):
    return context.aws_credentials
<|endoftext|>


<commit_before>VERSION = (1, 2, 2, 'rc1')

def get_version():
    return '%s.%s.%s-%s' % VERSION

try:
    from django.conf import settings
    import logging, os

    LOGFILE = os.path.join(settings.DIRNAME, 'axes.log')
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%a, %d %b %Y %H:%M:%S',
                        filename=LOGFILE,
                        filemode='w')

    fileLog = logging.FileHandler(LOGFILE, 'w')
    fileLog.setLevel(logging.DEBUG)

    # set a format which is simpler for console use
    formatter = logging.Formatter('%(asctime)s %(name)-12s: %(levelname)-8s %(message)s')

    # tell the handler to use this format
    fileLog.setFormatter(formatter)

    # add the handler to the root logger
    logging.getLogger('').addHandler(fileLog)
except:
    # if we have any problems, we most likely don't have a settings module loaded
    pass
<commit_msg>version bump<commit_after>VERSION = (1, 2, 3, 'rc1')

def get_version():
    return '%s.%s.%s-%s' % VERSION

try:
    from django.conf import settings
    import logging, os

    LOGFILE = os.path.join(settings.DIRNAME, 'axes.log')
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%a, %d %b %Y %H:%M:%S',
                        filename=LOGFILE,
                        filemode='w')

    fileLog = logging.FileHandler(LOGFILE, 'w')
    fileLog.setLevel(logging.DEBUG)

    # set a format which is simpler for console use
    formatter = logging.Formatter('%(asctime)s %(name)-12s: %(levelname)-8s %(message)s')

    # tell the handler to use this format
    fileLog.setFormatter(formatter)

    # add the handler to the root logger
    logging.getLogger('').addHandler(fileLog)
except:
    # if we have any problems, we most likely don't have a settings module loaded
    pass
<|endoftext|>


<commit_before>import tensorflow as tf
from tensorflow.python.ops import array_ops
from TFCommon.Initializer import gaussian_initializer, random_orthogonal_initializer
from six.moves import xrange

class BahdanauAttentionModule(object):
    """Attention Module
    Args:
        attention_units:    The attention module's capacity (should be proportional to query_units)
        memory:             A tensor, whose shape should be (None, Time, Unit)
        time_major:
    """
    def __init__(self, attention_units, memory, sequence_length=None, time_major=True, mode=0):
        self.attention_units    = attention_units
        self.enc_units          = memory.get_shape()[-1].value

        if time_major:
            memory = tf.transpose(memory, perm=(1,0,2))

        self.enc_length = tf.shape(memory)[1]
        self.batch_size = tf.shape(memory)[0]
        self.mode = mode
        self.mask = array_ops.sequence_mask(sequence_length, self.enc_length, tf.float32) if sequence_length is not None else None

        self.memory = tf.reshape(memory, (tf.shape(memory)[0], self.enc_length, 1, self.enc_units))
        ### pre-compute Uahj to minimize the computational cost
        with tf.variable_scope('attention'):
            Ua = tf.get_variable(name='Ua', shape=(1, 1, self.enc_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001) if self.mode == 0 else None)
        self.hidden_feats = tf.nn.conv2d(self.memory, Ua, [1,1,1,1], "SAME")

    def __call__(self, query):

        with tf.variable_scope('attention'):
            # Check if the memory's batch_size is consistent with query's batch_size

            query_units = query.get_shape()[-1].value

            Wa = tf.get_variable(name='Wa', shape=(query_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001) if self.mode == 0 else None)
            Va = tf.get_variable(name='Va', shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0) if self.mode == 0 else None)
            b  = tf.get_variable(name='b',  shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0) if self.mode == 0 else None)

            ### 1st. compute query_feat (query's repsentation in attention module)
            query_feat = tf.reshape(tf.matmul(query, Wa), (-1, 1, 1, self.attention_units))

            ### 2nd. compute the energy for all time steps in encoder (element-wise mul then reduce)
            e = tf.reduce_sum(Va * tf.nn.tanh(self.hidden_feats + query_feat + b), axis=(2,3))

            ### 3rd. compute the score
            #alpha = tf.nn.softmax(e)
            exp_e = tf.exp(e)
            if self.mask is not None:
                exp_e = exp_e * self.mask
            alpha = tf.divide(exp_e, tf.reduce_sum(exp_e, axis=-1, keep_dims=True))

            ### 4th. get the weighted context from memory (element-wise mul then reduce)
            context = tf.reshape(alpha, (tf.shape(query)[0], self.enc_length, 1, 1)) * self.memory
            context = tf.reduce_sum(context, axis=(1,2))

            return context, alpha

class LuongAttentionModule(object):
    """Attention Module
    Args:
        attention_units:    The attention module's capacity (should be proportional to query_units)
        memory:             A tensor, whose shape should be (None, Time, Unit)
        time_major:
    """
    def __init__(self, attention_units, memory, time_major=True):
        self.attention_units    = attention_units
        self.enc_units          = memory.get_shape()[-1].value

        if time_major:
            memory = tf.transpose(memory, perm=(1,0,2))

        self.enc_length = tf.shape(memory)[1]
        self.batch_size = tf.shape(memory)[0]

        self.memory = tf.reshape(memory, (tf.shape(memory)[0], self.enc_length, 1, self.enc_units))
        ### pre-compute Uahj to minimize the computational cost
        with tf.variable_scope('attention'):
            Ua = tf.get_variable(name='Ua', shape=(1, 1, self.enc_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001))
        self.hidden_feats = tf.nn.conv2d(self.memory, Ua, [1,1,1,1], "SAME")

    def __call__(self, query):

        with tf.variable_scope('attention'):
            # Check if the memory's batch_size is consistent with query's batch_size

            query_units = query.get_shape()[-1].value

            Wa = tf.get_variable(name='Wa', shape=(query_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001))
            Va = tf.get_variable(name='Va', shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0))
            b  = tf.get_variable(name='b',  shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0))

            ### 1st. compute query_feat (query's repsentation in attention module)
            query_feat = tf.reshape(tf.matmul(query, Wa), (-1, 1, 1, self.attention_units))

            ### 2nd. compute the energy for all time steps in encoder (element-wise mul then reduce)
            e = tf.reduce_sum(query_feat * self.hidden_feats, axis=(2,3))

            ### 3rd. compute the score
            alpha = tf.nn.softmax(e)

            ### 4th. get the weighted context from memory (element-wise mul then reduce)
            context = tf.reshape(alpha, (tf.shape(query)[0], self.enc_length, 1, 1)) * self.memory
            context = tf.reduce_sum(context, axis=(1,2))

            return context, alpha

<commit_msg>test softmax<commit_after>import tensorflow as tf
from tensorflow.python.ops import array_ops
from TFCommon.Initializer import gaussian_initializer, random_orthogonal_initializer
from six.moves import xrange

class BahdanauAttentionModule(object):
    """Attention Module
    Args:
        attention_units:    The attention module's capacity (should be proportional to query_units)
        memory:             A tensor, whose shape should be (None, Time, Unit)
        time_major:
    """
    def __init__(self, attention_units, memory, sequence_length=None, time_major=True, mode=0):
        self.attention_units    = attention_units
        self.enc_units          = memory.get_shape()[-1].value

        if time_major:
            memory = tf.transpose(memory, perm=(1,0,2))

        self.enc_length = tf.shape(memory)[1]
        self.batch_size = tf.shape(memory)[0]
        self.mode = mode
        self.mask = array_ops.sequence_mask(sequence_length, self.enc_length, tf.float32) if sequence_length is not None else None

        self.memory = tf.reshape(memory, (tf.shape(memory)[0], self.enc_length, 1, self.enc_units))
        ### pre-compute Uahj to minimize the computational cost
        with tf.variable_scope('attention'):
            Ua = tf.get_variable(name='Ua', shape=(1, 1, self.enc_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001) if self.mode == 0 else None)
        self.hidden_feats = tf.nn.conv2d(self.memory, Ua, [1,1,1,1], "SAME")

    def __call__(self, query):

        with tf.variable_scope('attention'):
            # Check if the memory's batch_size is consistent with query's batch_size

            query_units = query.get_shape()[-1].value

            Wa = tf.get_variable(name='Wa', shape=(query_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001) if self.mode == 0 else None)
            Va = tf.get_variable(name='Va', shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0) if self.mode == 0 else None)
            b  = tf.get_variable(name='b',  shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0) if self.mode == 0 else None)

            ### 1st. compute query_feat (query's repsentation in attention module)
            query_feat = tf.reshape(tf.matmul(query, Wa), (-1, 1, 1, self.attention_units))

            ### 2nd. compute the energy for all time steps in encoder (element-wise mul then reduce)
            e = tf.reduce_sum(Va * tf.nn.tanh(self.hidden_feats + query_feat + b), axis=(2,3))

            ### 3rd. compute the score
            if self.mask is not None:
                exp_e = tf.exp(e)
                exp_e = exp_e * self.mask
                alpha = tf.divide(exp_e, tf.reduce_sum(exp_e, axis=-1, keep_dims=True))
            else:
                alpha = tf.nn.softmax(e)

            ### 4th. get the weighted context from memory (element-wise mul then reduce)
            context = tf.reshape(alpha, (tf.shape(query)[0], self.enc_length, 1, 1)) * self.memory
            context = tf.reduce_sum(context, axis=(1,2))

            return context, alpha

class LuongAttentionModule(object):
    """Attention Module
    Args:
        attention_units:    The attention module's capacity (should be proportional to query_units)
        memory:             A tensor, whose shape should be (None, Time, Unit)
        time_major:
    """
    def __init__(self, attention_units, memory, time_major=True):
        self.attention_units    = attention_units
        self.enc_units          = memory.get_shape()[-1].value

        if time_major:
            memory = tf.transpose(memory, perm=(1,0,2))

        self.enc_length = tf.shape(memory)[1]
        self.batch_size = tf.shape(memory)[0]

        self.memory = tf.reshape(memory, (tf.shape(memory)[0], self.enc_length, 1, self.enc_units))
        ### pre-compute Uahj to minimize the computational cost
        with tf.variable_scope('attention'):
            Ua = tf.get_variable(name='Ua', shape=(1, 1, self.enc_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001))
        self.hidden_feats = tf.nn.conv2d(self.memory, Ua, [1,1,1,1], "SAME")

    def __call__(self, query):

        with tf.variable_scope('attention'):
            # Check if the memory's batch_size is consistent with query's batch_size

            query_units = query.get_shape()[-1].value

            Wa = tf.get_variable(name='Wa', shape=(query_units, self.attention_units),
                                 initializer=gaussian_initializer(mean=0.0, std=0.001))
            Va = tf.get_variable(name='Va', shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0))
            b  = tf.get_variable(name='b',  shape=(self.attention_units,),
                                 initializer=tf.constant_initializer(0.0))

            ### 1st. compute query_feat (query's repsentation in attention module)
            query_feat = tf.reshape(tf.matmul(query, Wa), (-1, 1, 1, self.attention_units))

            ### 2nd. compute the energy for all time steps in encoder (element-wise mul then reduce)
            e = tf.reduce_sum(query_feat * self.hidden_feats, axis=(2,3))

            ### 3rd. compute the score
            alpha = tf.nn.softmax(e)

            ### 4th. get the weighted context from memory (element-wise mul then reduce)
            context = tf.reshape(alpha, (tf.shape(query)[0], self.enc_length, 1, 1)) * self.memory
            context = tf.reduce_sum(context, axis=(1,2))

            return context, alpha

<|endoftext|>


<commit_before># Copyright (C) 2011 - Curtis Hovey <sinzui.is at verizon.net>
# This software is licensed under the MIT license (see the file COPYING).

from pocketlint.formatcheck import AnyTextChecker
from pocketlint.tests import (
    CheckerTestCase,
    data,
    )

class TestAnyTextMixin:
    """A mixin that provides common text tests."""

    def create_and_check(self, file_name, text):
        raise NotImplemented

    def test_without_conflicts(self):
        self.create_and_check('bogus', '<<<<<< look')
        self.assertEqual([], self.reporter.messages)

    def test_with_conflicts(self):
        self.create_and_check('bogus', '<<<<<<' + '<')
        self.assertEqual(
            [(1, 'File has conflicts.')], self.reporter.messages)

    def test_length_okay(self):
        self.create_and_check('bogus', 'short line')
        self.assertEqual([], self.reporter.messages)

    def test_long_length(self):
        long_line = '1234 56189' * 8
        self.create_and_check('bogus', long_line)
        self.assertEqual(
            [(1, 'Line exceeds 78 characters.')],
            self.reporter.messages)

    def test_no_trailing_whitespace(self):
        self.create_and_check('bogus', 'no trailing white-space')
        self.assertEqual([], self.reporter.messages)

    def test_trailing_whitespace(self):
        self.create_and_check('bogus', 'trailing white-space ')
        self.assertEqual(
            [(1, 'Line has trailing whitespace.')],
            self.reporter.messages)

    def test_without_tabs(self):
        self.create_and_check('bogus', '    without tabs')
        self.assertEqual([], self.reporter.messages)

    def test_with_tabs(self):
        self.create_and_check('bogus', '\twith tabs')
        self.assertEqual(
            [(1, 'Line contains a tab character.')],
            self.reporter.messages)

class TestText(CheckerTestCase, TestAnyTextMixin):
    """Verify text integration."""

    def create_and_check(self, file_name, text):
        checker = AnyTextChecker(file_name, text, self.reporter)
        checker.check()

    def test_with_tabs(self):
        """Text files may contain tabs.."""
        pass
<commit_msg>Removed unused import.<commit_after># Copyright (C) 2011 - Curtis Hovey <sinzui.is at verizon.net>
# This software is licensed under the MIT license (see the file COPYING).

from pocketlint.formatcheck import AnyTextChecker
from pocketlint.tests import CheckerTestCase

class TestAnyTextMixin:
    """A mixin that provides common text tests."""

    def create_and_check(self, file_name, text):
        raise NotImplemented

    def test_without_conflicts(self):
        self.create_and_check('bogus', '<<<<<< look')
        self.assertEqual([], self.reporter.messages)

    def test_with_conflicts(self):
        self.create_and_check('bogus', '<<<<<<' + '<')
        self.assertEqual(
            [(1, 'File has conflicts.')], self.reporter.messages)

    def test_length_okay(self):
        self.create_and_check('bogus', 'short line')
        self.assertEqual([], self.reporter.messages)

    def test_long_length(self):
        long_line = '1234 56189' * 8
        self.create_and_check('bogus', long_line)
        self.assertEqual(
            [(1, 'Line exceeds 78 characters.')],
            self.reporter.messages)

    def test_no_trailing_whitespace(self):
        self.create_and_check('bogus', 'no trailing white-space')
        self.assertEqual([], self.reporter.messages)

    def test_trailing_whitespace(self):
        self.create_and_check('bogus', 'trailing white-space ')
        self.assertEqual(
            [(1, 'Line has trailing whitespace.')],
            self.reporter.messages)

    def test_without_tabs(self):
        self.create_and_check('bogus', '    without tabs')
        self.assertEqual([], self.reporter.messages)

    def test_with_tabs(self):
        self.create_and_check('bogus', '\twith tabs')
        self.assertEqual(
            [(1, 'Line contains a tab character.')],
            self.reporter.messages)

class TestText(CheckerTestCase, TestAnyTextMixin):
    """Verify text integration."""

    def create_and_check(self, file_name, text):
        checker = AnyTextChecker(file_name, text, self.reporter)
        checker.check()

    def test_with_tabs(self):
        """Text files may contain tabs.."""
        pass
<|endoftext|>