<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.utils import flt, cstr
from frappe import msgprint, _

def execute(filters=None):
	if not filters: filters = {}

	salary_slips = get_salary_slips(filters)
	columns, earning_types, ded_types = get_columns(salary_slips)
	ss_earning_map = get_ss_earning_map(salary_slips)
	ss_ded_map = get_ss_ded_map(salary_slips)

	data = []
	for ss in salary_slips:
		row = [ss.name, ss.employee, ss.employee_name, ss.branch, ss.department, ss.designation,
			ss.company, ss.month, ss.leave_withut_pay, ss.payment_days]

		for e in earning_types:
			row.append(ss_earning_map.get(ss.name, {}).get(e))

		row += [ss.arrear_amount, ss.leave_encashment_amount, ss.gross_pay]

		for d in ded_types:
			row.append(ss_ded_map.get(ss.name, {}).get(d))

		row += [ss.total_deduction, ss.net_pay]

		data.append(row)

	return columns, data

def get_columns(salary_slips):
	columns = [
		_("Salary Slip ID") + ":Link/Salary Slip:150",_("Employee") + ":Link/Employee:120", _("Employee Name") + "::140", _("Branch") + ":Link/Branch:120",
		_("Department") + ":Link/Department:120", _("Designation") + ":Link/Designation:120",
		 _("Company") + ":Link/Company:120", _("Month") + "::80", _("Leave Without Pay") + ":Float:130",
		_("Payment Days") + ":Float:120"
	]

	earning_types = frappe.db.sql_list("""select distinct salary_component from `tabSalary Detail`
		where amount != 0 and parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]))

	ded_types = frappe.db.sql_list("""select distinct salary_component from `tabSalary Detail`
		where amount != 0 and parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]))

	columns = columns + [(e + ":Currency:120") for e in earning_types] + \
		["Arrear Amount:Currency:120", "Leave Encashment Amount:Currency:150",
		"Gross Pay:Currency:120"] + [(d + ":Currency:120") for d in ded_types] + \
		["Total Deduction:Currency:120", "Net Pay:Currency:120"]

	return columns, earning_types, ded_types

def get_salary_slips(filters):
	conditions, filters = get_conditions(filters)
	salary_slips = frappe.db.sql("""select * from `tabSalary Slip` where docstatus = 1 %s
		order by employee, month""" % conditions, filters, as_dict=1)

	if not salary_slips:
		msgprint(_("No salary slip found for month: ") + cstr(filters.get("month")) +
			_(" and year: ") + cstr(filters.get("fiscal_year")), raise_exception=1)

	return salary_slips

def get_conditions(filters):
	conditions = ""
	if filters.get("month"):
		month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov",
			"Dec"].index(filters["month"]) + 1
		filters["month"] = month
		conditions += " and month = %(month)s"

	if filters.get("fiscal_year"): conditions += " and fiscal_year = %(fiscal_year)s"
	if filters.get("company"): conditions += " and company = %(company)s"
	if filters.get("employee"): conditions += " and employee = %(employee)s"

	return conditions, filters

def get_ss_earning_map(salary_slips):
	ss_earnings = frappe.db.sql("""select parent, salary_component, amount
		from `tabSalary Detail` where parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]), as_dict=1)

	ss_earning_map = {}
	for d in ss_earnings:
		ss_earning_map.setdefault(d.parent, frappe._dict()).setdefault(d.salary_component, [])
		ss_earning_map[d.parent][d.salary_component] = flt(d.amount)

	return ss_earning_map

def get_ss_ded_map(salary_slips):
	ss_deductions = frappe.db.sql("""select parent, salary_component, amount
		from `tabSalary Detail` where parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]), as_dict=1)

	ss_ded_map = {}
	for d in ss_deductions:
		ss_ded_map.setdefault(d.parent, frappe._dict()).setdefault(d.salary_component, [])
		ss_ded_map[d.parent][d.salary_component] = flt(d.amount)

	return ss_ded_map<commit_msg>[fix] duplicate columns in monthly salary report #6623<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.utils import flt, cstr
from frappe import msgprint, _

def execute(filters=None):
	if not filters: filters = {}

	salary_slips = get_salary_slips(filters)
	columns, earning_types, ded_types = get_columns(salary_slips)
	ss_earning_map = get_ss_earning_map(salary_slips)
	ss_ded_map = get_ss_ded_map(salary_slips)

	data = []
	for ss in salary_slips:
		row = [ss.name, ss.employee, ss.employee_name, ss.branch, ss.department, ss.designation,
			ss.company, ss.month, ss.leave_withut_pay, ss.payment_days]

		for e in earning_types:
			row.append(ss_earning_map.get(ss.name, {}).get(e))

		row += [ss.arrear_amount, ss.leave_encashment_amount, ss.gross_pay]

		for d in ded_types:
			row.append(ss_ded_map.get(ss.name, {}).get(d))

		row += [ss.total_deduction, ss.net_pay]

		data.append(row)

	return columns, data

def get_columns(salary_slips):
	columns = [
		_("Salary Slip ID") + ":Link/Salary Slip:150",_("Employee") + ":Link/Employee:120", _("Employee Name") + "::140", _("Branch") + ":Link/Branch:120",
		_("Department") + ":Link/Department:120", _("Designation") + ":Link/Designation:120",
		_("Company") + ":Link/Company:120", _("Month") + "::80", _("Leave Without Pay") + ":Float:130",
		_("Payment Days") + ":Float:120"
	]

	salary_components = {_("Earning"): [], _("Deduction"): []}

	for component in frappe.db.sql("""select distinct sd.salary_component, sc.type
		from `tabSalary Detail` sd, `tabSalary Component` sc
		where sc.name=sd.salary_component and sd.amount != 0 and sd.parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]), as_dict=1):
		salary_components[component.type].append(component.salary_component)

	columns = columns + [(e + ":Currency:120") for e in salary_components[_("Earning")]] + \
		[ _("Arrear Amount") + ":Currency:120", _("Leave Encashment Amount") + ":Currency:150",
		_("Gross Pay") + ":Currency:120"] + [(d + ":Currency:120") for d in salary_components[_("Deduction")]] + \
		[_("Total Deduction") + ":Currency:120", _("Net Pay") + ":Currency:120"]

	return columns, salary_components[_("Earning")], salary_components[_("Deduction")]

def get_salary_slips(filters):
	conditions, filters = get_conditions(filters)
	salary_slips = frappe.db.sql("""select * from `tabSalary Slip` where docstatus = 1 %s
		order by employee, month""" % conditions, filters, as_dict=1)

	if not salary_slips:
		msgprint(_("No salary slip found for month: ") + cstr(filters.get("month")) +
			_(" and year: ") + cstr(filters.get("fiscal_year")), raise_exception=1)

	return salary_slips

def get_conditions(filters):
	conditions = ""
	if filters.get("month"):
		month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov",
			"Dec"].index(filters["month"]) + 1
		filters["month"] = month
		conditions += " and month = %(month)s"

	if filters.get("fiscal_year"): conditions += " and fiscal_year = %(fiscal_year)s"
	if filters.get("company"): conditions += " and company = %(company)s"
	if filters.get("employee"): conditions += " and employee = %(employee)s"

	return conditions, filters

def get_ss_earning_map(salary_slips):
	ss_earnings = frappe.db.sql("""select parent, salary_component, amount
		from `tabSalary Detail` where parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]), as_dict=1)

	ss_earning_map = {}
	for d in ss_earnings:
		ss_earning_map.setdefault(d.parent, frappe._dict()).setdefault(d.salary_component, [])
		ss_earning_map[d.parent][d.salary_component] = flt(d.amount)

	return ss_earning_map

def get_ss_ded_map(salary_slips):
	ss_deductions = frappe.db.sql("""select parent, salary_component, amount
		from `tabSalary Detail` where parent in (%s)""" %
		(', '.join(['%s']*len(salary_slips))), tuple([d.name for d in salary_slips]), as_dict=1)

	ss_ded_map = {}
	for d in ss_deductions:
		ss_ded_map.setdefault(d.parent, frappe._dict()).setdefault(d.salary_component, [])
		ss_ded_map[d.parent][d.salary_component] = flt(d.amount)

	return ss_ded_map<|endoftext|>


<commit_before>import wx
import wx.xrc as xrc
import pkg_resources
import numpy
import motmot.imops.imops as imops
import pyglet.gl as gl

from pygarrayimage.arrayimage import ArrayInterfaceImage

import motmot.wxglvideo.wxglvideo as vid

class PointDisplayCanvas( vid.DynamicImageCanvas ):
    def __init__(self,*args,**kw):
        super(PointDisplayCanvas, self).__init__(*args,**kw)
        self.extra_points_linesegs = ([], [])

    def core_draw(self):
        super(PointDisplayCanvas, self).core_draw()

        points,linesegs = self.extra_points_linesegs
        gl.glColor4f(0.0,1.0,0.0,1.0) # green point

        gl.glBegin(gl.GL_POINTS)
        for pt in points:
            gl.glVertex2f(pt[0],pt[1])
        gl.glEnd()

        gl.glBegin(gl.GL_LINES)
        for (x0,y0,x1,y1) in linesegs:
            gl.glVertex2f(x0,y0)
            gl.glVertex2f(x1,y1)
        gl.glEnd()

        gl.glColor4f(1.0,1.0,1.0,1.0) # green point

    def extra_initgl(self):
        gl.glEnable( gl.GL_POINT_SMOOTH )
        gl.glPointSize(5)

class DynamicImageCanvas(wx.Panel):
    def __init__(self,*args,**kw):
        super(DynamicImageCanvas, self).__init__(*args,**kw)

        self.rotate_180 = False
        self.flip_lr = False

        self.children = {}
        self.green_points = {}
        self.red_points = {}
        self.lbrt = {}

        self.box = wx.BoxSizer(wx.HORIZONTAL)
        self.SetSizer(self.box)
        #wx.EVT_IDLE( self, self.OnIdle )

    def _new_child(self,id_val,image):
        child = PointDisplayCanvas(self,-1)
        child.set_fullcanvas(True)
        self.box.Add( child, 1, wx.EXPAND)
        self.Layout()
        pygim = ArrayInterfaceImage( image, allow_copy=False )
        child.new_image( pygim )
        child.set_rotate_180( self.rotate_180 )
        child.set_flip_lr( self.flip_lr )

        self.children[id_val] = child
        self.green_points[id_val] = []
        self.red_points[id_val] = []
        self.lbrt[id_val] = ()

    def set_rotate_180(self, value):
        self.rotate_180 = value
        for id_val in self.children:
            child = self.children[id_val]
            child.set_rotate_180(value)

    def set_flip_LR(self, value):
        self.flip_lr = value
        for id_val in self.children:
            child = self.children[id_val]
            child.set_flip_lr(value)

    def set_green_points(self,id_val,points):
        self.green_points[id_val]=points

    def set_red_points(self,id_val,points):
        self.red_points[id_val]=points

    def set_lbrt(self,id_val,lbrt):
        self.lbrt[id_val]=lbrt

    def update_image(self, id_val, image, format='MONO8',
                     xoffset=0, yoffset=0):
        image=numpy.asarray(image)
        if format == 'RGB8':
            image = imops.rgb8_to_rgb8( image )
        elif format == 'ARGB8':
            image = imops.argb8_to_rgb8( image )
        elif format == 'YUV411':
            image = imops.yuv411_to_rgb8( image )
        elif format == 'YUV422':
            image = imops.yuv422_to_rgb8( image )
        elif format == 'MONO8':
            pass
        elif format == 'RAW8':
            pass
        elif format == 'MONO16':
            image = imops.mono16_to_mono8_middle8bits( image )
        else:
            raise ValueError("Unknown format '%s'"%(format,))

        if id_val not in self.children:
            # The line gives us:
            #  Gtk-CRITICAL **: gtk_widget_set_colormap: assertion `!GTK_WIDGET_REALIZED (widget)' failed
            self._new_child(id_val,image)
        else:
            child = self.children[id_val]
            child.update_image(image)
            #child.extra_points_linesegs = ([],[])

    def update_image_and_drawings(self,
                                  id_val,
                                  image,
                                  format='MONO8',
                                  points=None,
                                  linesegs=None,
                                  xoffset=0,
                                  yoffset=0):

        try:
            child = self.children[id_val]
        except KeyError:
            # XXX BUG: on the first frame for this camera, no points will be drawn
            pass
        else:
            child.extra_points_linesegs = points, linesegs

        self.update_image( id_val,
                           image,
                           format=format,
                           xoffset=xoffset,
                           yoffset=yoffset)
        # draw points and linesegs
        if points is None:
            points = []
        if linesegs is None:
            linesegs = []

    def OnIdle(self, event):
        for id_val in self.children:
            child = self.children[id_val]
            child.OnDraw()
        event.RequestMore( True )
<commit_msg>actually draw red points<commit_after>import wx
import wx.xrc as xrc
import pkg_resources
import numpy
import motmot.imops.imops as imops
import pyglet.gl as gl

from pygarrayimage.arrayimage import ArrayInterfaceImage

import motmot.wxglvideo.wxglvideo as vid

class PointDisplayCanvas( vid.DynamicImageCanvas ):
    def __init__(self,*args,**kw):
        super(PointDisplayCanvas, self).__init__(*args,**kw)
        self.extra_points_linesegs = ([], [])
        self.red_points = None

    def core_draw(self):
        super(PointDisplayCanvas, self).core_draw()

        points,linesegs = self.extra_points_linesegs
        gl.glColor4f(0.0,1.0,0.0,1.0) # green point

        gl.glBegin(gl.GL_POINTS)
        for pt in points:
            gl.glVertex2f(pt[0],pt[1])
        gl.glEnd()

        gl.glBegin(gl.GL_LINES)
        for (x0,y0,x1,y1) in linesegs:
            gl.glVertex2f(x0,y0)
            gl.glVertex2f(x1,y1)
        gl.glEnd()

        if self.red_points is not None:
            gl.glColor4f(1.0,0.0,0.0,1.0)
            gl.glBegin(gl.GL_POINTS)
            for pt in self.red_points:
                gl.glVertex2f(pt[0],pt[1])
            gl.glEnd()

        gl.glColor4f(1.0,1.0,1.0,1.0) # restore white color

    def extra_initgl(self):
        gl.glEnable( gl.GL_POINT_SMOOTH )
        gl.glPointSize(5)

class DynamicImageCanvas(wx.Panel):
    def __init__(self,*args,**kw):
        super(DynamicImageCanvas, self).__init__(*args,**kw)

        self.rotate_180 = False
        self.flip_lr = False

        self.children = {}
        self.lbrt = {}

        self.box = wx.BoxSizer(wx.HORIZONTAL)
        self.SetSizer(self.box)
        #wx.EVT_IDLE( self, self.OnIdle )

    def _new_child(self,id_val,image):
        child = PointDisplayCanvas(self,-1)
        child.set_fullcanvas(True)
        self.box.Add( child, 1, wx.EXPAND)
        self.Layout()
        pygim = ArrayInterfaceImage( image, allow_copy=False )
        child.new_image( pygim )
        child.set_rotate_180( self.rotate_180 )
        child.set_flip_lr( self.flip_lr )

        self.children[id_val] = child
        self.lbrt[id_val] = ()

    def set_rotate_180(self, value):
        self.rotate_180 = value
        for id_val in self.children:
            child = self.children[id_val]
            child.set_rotate_180(value)

    def set_flip_LR(self, value):
        self.flip_lr = value
        for id_val in self.children:
            child = self.children[id_val]
            child.set_flip_lr(value)

    def set_red_points(self,id_val,points):
        self.children[id_val].red_points=points

    def set_lbrt(self,id_val,lbrt):
        self.lbrt[id_val]=lbrt

    def update_image(self, id_val, image, format='MONO8',
                     xoffset=0, yoffset=0):
        image=numpy.asarray(image)
        if format == 'RGB8':
            image = imops.rgb8_to_rgb8( image )
        elif format == 'ARGB8':
            image = imops.argb8_to_rgb8( image )
        elif format == 'YUV411':
            image = imops.yuv411_to_rgb8( image )
        elif format == 'YUV422':
            image = imops.yuv422_to_rgb8( image )
        elif format == 'MONO8':
            pass
        elif format == 'RAW8':
            pass
        elif format == 'MONO16':
            image = imops.mono16_to_mono8_middle8bits( image )
        else:
            raise ValueError("Unknown format '%s'"%(format,))

        if id_val not in self.children:
            # The line gives us:
            #  Gtk-CRITICAL **: gtk_widget_set_colormap: assertion `!GTK_WIDGET_REALIZED (widget)' failed
            self._new_child(id_val,image)
        else:
            child = self.children[id_val]
            child.update_image(image)
            #child.extra_points_linesegs = ([],[])

    def update_image_and_drawings(self,
                                  id_val,
                                  image,
                                  format='MONO8',
                                  points=None,
                                  linesegs=None,
                                  xoffset=0,
                                  yoffset=0):

        try:
            child = self.children[id_val]
        except KeyError:
            # XXX BUG: on the first frame for this camera, no points will be drawn
            pass
        else:
            child.extra_points_linesegs = (points, linesegs)

        self.update_image( id_val,
                           image,
                           format=format,
                           xoffset=xoffset,
                           yoffset=yoffset)
        # draw points and linesegs
        if points is None:
            points = []
        if linesegs is None:
            linesegs = []

    def OnIdle(self, event):
        for id_val in self.children:
            child = self.children[id_val]
            child.OnDraw()
        event.RequestMore( True )
<|endoftext|>


<commit_before>import json
from htmlgen import Document, Division, Image

def ParseCollection(info):
    div = Division()
    div.add_css_classes('dataset')

    div.append(createPlainElement('path', info))
    div.append(createPlainElement('name', info))
    div.append(createPlainElement('license', info))
    div.append(createPlainElement('description', info))
    div.append(createPlainElement('acknowledgements', info))
    div.append(createTagElements(info))
    div.append(createFileElements(info))
    div.append(createThumbnailElements(info))
    div.append(createSoftwareElements(info))
    div.append(createCitationElements(info))

    return div

####################################################################################
####################################################################################
####################################################################################

def createPlainElement(type, info):
    div = Division()
    div.add_css_classes('dataset-' + type)

    div.append(info[type])

    return div

def createTagElements(info):
    div = Division()
    div.add_css_classes('dataset-tags')

    tags = info['tags']
    for t in tags:
        d = Division()
        d.add_css_classes('dataset-tag')
        d.id = t
        d.append(t)
        div.append(d)

    return div

def createFileElements(info):
    div = Division()
    div.add_css_classes('dataset-files')

    files = info['files']
    for file in files:
        d = Division()
        d.add_css_classes('dataset-file')

        # Name
        nameDiv = Division()
        nameDiv.add_css_classes('dataset-file-name')
        nameDiv.append(file['name'])
        d.append(nameDiv)

        # Description
        descDiv = Division()
        descDiv.add_css_classes('dataset-file-description')
        descDiv.append(file['description'])
        d.append(descDiv)

        # Format
        formatDiv = Division()
        formatDiv.add_css_classes('dataset-file-format')
        formatDiv.append(file['format'])
        d.append(formatDiv)

        # Resolution
        resolutionDiv = Division()
        resolutionDiv.add_css_classes('dataset-file-resolution')
        resolutionDiv.append(str(file['resolution']))
        d.append(resolutionDiv)

        div.append(d)

    return div

def createThumbnailElements(info):
    div = Division()
    div.add_css_classes('dataset-thumbnails')

    thumbnails = info['thumbnails']

    for thumbnail in thumbnails:
        d = Division()
        d.add_css_classes('dataset-thumbnail')

        # Image
        image = Image(thumbnail['name'], thumbnail['caption'])
        d.append(image)

        # Name
        nameDiv = Division()
        nameDiv.add_css_classes('dataset-thumbnail-name')
        nameDiv.append(thumbnail['name'])
        d.append(nameDiv);

        # Caption
        captionDiv = Division()
        captionDiv.add_css_classes('dataset-thumbnail-caption')
        captionDiv.append(thumbnail['caption'])
        d.append(captionDiv)

        div.append(d)

    return div

def createSoftwareElements(info):
    div = Division()
    div.add_css_classes('dataset-softwares')

    softwares = info['software']
    for software in softwares:
        d = Division()
        d.add_css_classes('dataset-software')

        allSoftwares = info['allSoftwares']

        s = allSoftwares[software]

        # Name
        nameDiv = Division()
        nameDiv.add_css_classes('dataset-software-name')

        nameDiv.append(s['name'])
        d.append(nameDiv)

        # URL
        urlDiv = Division()
        urlDiv.add_css_classes('dataset-software-url')

        urlDiv.append(s['url'])
        d.append(urlDiv)

        div.append(d)

    return div

def createCitationElements(info):
    div = Division()
    div.add_css_classes('dataset-citations')

    citations = info['citations']
    for citation in citations:
        type = citation['type']
        payload = citation['payload']

        d = {
          'plain': createPlainCitationElement,
          'DOI': createDOICitationElement,
          'BibTex': createBibTexCitationElement
        }[type](payload)

        div.append(d)

    return div

####################################################################################
####################################################################################
####################################################################################

def createPlainCitationElement(payload):
    div = Division()
    div.add_css_classes('dataset-citation-plain')
    div.append(payload)

    return div

def createDOICitationElement(payload):
    div = Division()
    div.add_css_classes('dataset-citation-doi')
    div.append(payload)

    return div

def createBibTexCitationElement(payload):
    div = Division()
    div.add_css_classes('dataset-citation-bibtex')
    div.append(payload)

    return div

####################################################################################
####################################################################################
####################################################################################

def generate(meta, software):
    doc = Document()

    for key, value in meta.items():
        value['path'] = key
        value['allSoftwares'] = software
        d = ParseCollection(value)
        doc.append_body(d)

    return doc<commit_msg>Renamed plain element to text element<commit_after>import json
from htmlgen import Document, Division, Image

def ParseCollection(info):
    div = Division()
    div.add_css_classes('dataset')

    div.append(createTextElement('path', info))
    div.append(createTextElement('name', info))
    div.append(createTextElement('license', info))
    div.append(createTextElement('description', info))
    div.append(createTextElement('acknowledgements', info))
    div.append(createTagElements(info))
    div.append(createFileElements(info))
    div.append(createThumbnailElements(info))
    div.append(createSoftwareElements(info))
    div.append(createCitationElements(info))
    div.append(createTextElement('notes', info))

    return div

####################################################################################
####################################################################################
####################################################################################

def createTextElement(type, info):
    div = Division()
    div.add_css_classes('dataset-' + type)

    div.append(info[type])

    return div

def createTagElements(info):
    div = Division()
    div.add_css_classes('dataset-tags')

    tags = info['tags']
    for t in tags:
        d = Division()
        d.add_css_classes('dataset-tag')
        d.id = t
        d.append(t)
        div.append(d)

    return div

def createFileElements(info):
    div = Division()
    div.add_css_classes('dataset-files')

    files = info['files']
    for file in files:
        d = Division()
        d.add_css_classes('dataset-file')

        # Name
        nameDiv = Division()
        nameDiv.add_css_classes('dataset-file-name')
        nameDiv.append(file['name'])
        d.append(nameDiv)

        # Description
        descDiv = Division()
        descDiv.add_css_classes('dataset-file-description')
        descDiv.append(file['description'])
        d.append(descDiv)

        # Format
        formatDiv = Division()
        formatDiv.add_css_classes('dataset-file-format')
        formatDiv.append(file['format'])
        d.append(formatDiv)

        # Resolution
        resolutionDiv = Division()
        resolutionDiv.add_css_classes('dataset-file-resolution')
        resolutionDiv.append(str(file['resolution']))
        d.append(resolutionDiv)

        div.append(d)

    return div

def createThumbnailElements(info):
    div = Division()
    div.add_css_classes('dataset-thumbnails')

    thumbnails = info['thumbnails']

    for thumbnail in thumbnails:
        d = Division()
        d.add_css_classes('dataset-thumbnail')

        # Image
        image = Image(thumbnail['name'], thumbnail['caption'])
        d.append(image)

        # Name
        nameDiv = Division()
        nameDiv.add_css_classes('dataset-thumbnail-name')
        nameDiv.append(thumbnail['name'])
        d.append(nameDiv);

        # Caption
        captionDiv = Division()
        captionDiv.add_css_classes('dataset-thumbnail-caption')
        captionDiv.append(thumbnail['caption'])
        d.append(captionDiv)

        div.append(d)

    return div

def createSoftwareElements(info):
    div = Division()
    div.add_css_classes('dataset-softwares')

    softwares = info['software']
    for software in softwares:
        d = Division()
        d.add_css_classes('dataset-software')

        allSoftwares = info['allSoftwares']

        s = allSoftwares[software]

        # Name
        nameDiv = Division()
        nameDiv.add_css_classes('dataset-software-name')

        nameDiv.append(s['name'])
        d.append(nameDiv)

        # URL
        urlDiv = Division()
        urlDiv.add_css_classes('dataset-software-url')

        urlDiv.append(s['url'])
        d.append(urlDiv)

        div.append(d)

    return div

def createCitationElements(info):
    div = Division()
    div.add_css_classes('dataset-citations')

    citations = info['citations']
    for citation in citations:
        type = citation['type']
        payload = citation['payload']

        d = {
          'plain': createPlainCitationElement,
          'DOI': createDOICitationElement,
          'BibTex': createBibTexCitationElement
        }[type](payload)

        div.append(d)

    return div

####################################################################################
####################################################################################
####################################################################################

def createPlainCitationElement(payload):
    div = Division()
    div.add_css_classes('dataset-citation-plain')
    div.append(payload)

    return div

def createDOICitationElement(payload):
    div = Division()
    div.add_css_classes('dataset-citation-doi')
    div.append(payload)

    return div

def createBibTexCitationElement(payload):
    div = Division()
    div.add_css_classes('dataset-citation-bibtex')
    div.append(payload)

    return div

####################################################################################
####################################################################################
####################################################################################

def generate(meta, software):
    doc = Document()

    for key, value in meta.items():
        value['path'] = key
        value['allSoftwares'] = software
        d = ParseCollection(value)
        doc.append_body(d)

    return doc<|endoftext|>


<commit_before>import copy
import json
import random
import uuid

from twisted.logger import Logger

from . import version

class Game(object):
  log = Logger()

  def __init__(self, factory):
    self.factory = factory
    self.black_cards = []
    self.database_hash = None
    self.name = ''
    self.open = True
    self.password_hash = None
    self.running = False
    self.uuid = None
    self.users = []
    self.white_cards = []

  @classmethod
  def create(cls, factory, name, password_hash = None):
    game = cls(factory)
    game.database_hash = factory.card_database.hash
    game.name = name
    game.password_hash = password_hash
    game.uuid = uuid.uuid4()
    game.loadCards()
    return game

  @classmethod
  def load(cls, factory, **data):
    game = cls(factory)
    game.name = data['name']
    game.password_hash = data['password_hash'] if len(data['password_hash']) else None
    game.open = False
    game.database_hash = data['database_hash']
    game.uuid = uuid.UUID(data['id'])

    game.users = json.loads(data['users'])
    for user in game.users:
      user['joined'] = False
      user['chosen_cards'] = []
      user['white_cards'] = [factory.card_database.getCard(c) for c in user['white_cards']]

    game.white_cards = [factory.card_database.getCard(c) for c in json.loads(data['cards'])['white_cards']]
    game.black_cards = [factory.card_database.getCard(c) for c in json.loads(data['cards'])['black_cards']]

    return game

  def loadCards(self):

    if len(self.black_cards) == 0:
      self.black_cards = self.factory.card_database.getBlackCards()
      random.shuffle(self.black_cards)

    if len(self.white_cards) == 0:
      self.white_cards = self.factory.card_database.getWhiteCards()

      # we need to strip all white cards users currently have in their pile
      for user in self.users:
        for card in user['white_cards']:
          del self.white_cards[self.white_cards.index(card)]

      random.shuffle(self.white_cards)

  def mayJoin(self, user):
    if self.running:
      return self.formatted(join = False, message = 'game already running')
    if self.open:
      if len(self.users) + 1 > self.factory.card_database.max_players_per_game:
        return self.formatted(join=False, message='no more players allowed')
      return self.formatted(join=True)
    else:
      possible_users = [u for u in self.users if u['user'] == user.id]

      if len(possible_users) != 1:
        return self.formatted(join = False, message = 'you are no member of this paused game')

      if possible_users[0]['joined']:
        return self.formatted(join = False, message = 'already joined this game')

      return self.formatted(join = True)

  def join(self, user, password):
    joinable = self.mayJoin(user)
    if not joinable['join']:
      return self.formatted(success=False, message=joinable['message'])
    if self.protected and password != self.password_hash:
      return self.formatted(success=False, message='wrong password supplied')

    if self.open:
      self.users.append(self.userdict(user, len(self.users) == 0))
      user.setGame(self)
    else:
      possible_users = [u for u in self.users if u['user'] == user.id and not u['joined']]
      possible_users[0]['joined'] = True
      user.setGame(self)

    return self.formatted(success=True, game_id = self.id)

  def getAllUsers(self):
    return [self.factory.findUser(u['user']) for u in self.users if u['joined']]

  def start(self):
    if len(self.getAllUsers())<3:
      return self.formatted(success=False, message='not enough players in this game')

    if self.running:
      return self.formatted(success=False, message='already running')

    # if the game is currently open, we need to shuffle the users
    if self.open:
      random.shuffle(self.users)
      self.open = False

    self.open = False
    self.running = True

    # all users need to get 10 cards
    for i in range(len(self.users)*10):
      self.users[i/10]['white_cards'].append(self.white_cards[i])
    self.white_cards=self.white_cards[len(self.users)*10:]

    # determine the one with the black card
    # index at 0 will always be the czar
    # and black_cards 0 will always be the current black card

    return self.formatted(success=True)

  def getCurrentBlackCard(self):
    try:
      return self.black_cards[0]
    except IndexError:
      return None

  def getAllWhiteCardsForUsers(self):
    return [(self.factory.findUser(self.users[i]['user']), self.users[i]['white_cards']) for i in range(len(self.users))]

  def suspend(self, user):

    self.pause()

    if user.getGame() is not self:
      self.log.warn('user {user} not in game {game}', user = user.id, game = self.id)
      return

    possible_users = [u for u in self.users if u['user'] == user.id and u['joined']]
    if len(possible_users) != 1:
      self.log.warn('found {count} users in game {game} while suspending due to user {user}', count = len(possible_users), game = self.id, user = user.id)
    else:
      possible_users[0]['joined'] = False
      user.setGame(None)
      self.log.info('user {user} suspended game {game}', user = user.id, game = self.id)

  def leave(self, user):
    # forces the user to leave

    if user.getGame() is not self:
      return self.formatted(success = False, message = 'user is not in this game')

    if len(self.getAllUsers()) == 0:
      self.log.warn('no users in this game, {user} tried to leave', user = user.id)
      return self.formatted(success = False, message = 'no users found in this game')

    possible_users = [u for u in self.users if u['user'] == user.id and u['joined']]

    if len(possible_users) != 1:
      self.log.warn('{user} tried to leave game {game}, but found {count} possible users', user = user.id, game = self.id, count = len(possible_users))
      return self.formatted(success = False, message = 'unable to find user in this game')

    if possible_users[0]['creator']:
      for u in self.users:
        if u != possible_users[0] and not u['creator']:
          u['creator'] = True
          break

    del self.users[self.users.index(possible_users[0])]
    user.setGame(None)

    self.log.info('user {user} left game {game}', user = user.id, game = self.id)

    self.pause()

    if len(self.users)<3:
      self.open = True
      for user in self.users:
        user['white_cards'] = []
        user['chosen_cards'] = []
        user['black_cards'] = 0
      self.white_cards = []
      self.black_cards = []
      self.loadCards()

      self.log.info('not enough users left, game opened up all new')

    return self.formatted(success = True, unlinked = self.unlink())

  def pause(self):

    if not self.running:
      return

    white_cards = []

    self.running = False

    for user in self.users:
      user['chosen_cards'] = []
      white_cards += user['white_cards']
      user['white_cards'] = []

    white_cards += self.white_cards

    self.white_cards = white_cards

    self.log.info('game {game} paused', game = self.id)

  def unlink(self, force = False):
    # can only unlink if no users are left
    # as long as it is not forced (force by using the delete button)
    if len(self.users)>0 and not force:
      return False

    self.factory.unlinkGame(self)
    self.log.info('game {game} deleted', game = self.id)
    return True

  def pack(self):
    # saves the current game into the database
    # at first we construct the sql arguments dict

    args = {}

    args['id'] = self.uuid.hex
    args['name'] = self.name

    users = copy.deepcopy(self.users)
    for user in users:
      del user['joined']
      del user['chosen_cards']
      user['white_cards'] = [c.id for c in user['white_cards']]

    args['users'] = json.dumps(users)

    black_cards = [c.id for c in self.black_cards]
    white_cards = [c.id for c in self.white_cards]

    args['cards'] = json.dumps({'white_cards': white_cards, 'black_cards': black_cards})

    args['password_hash'] = self.password_hash if self.protected else ''
    args['database_hash'] = self.database_hash
    args['server_version_major'] = version.MAJOR
    args['server_version_minor'] = version.MINOR
    args['server_version_revision'] = version.REVISION

    return args

  def chooseCards(self, user, cards):
    d = [u for u in self.users if u['user'] == user.id]

    if len(d) != 1:
      self.log.warn('user {user} wants to choose cards, but user not found in game {game}', user = user.id, game = self.id)
      return self.formatted(success = False, message = 'user not in this game')

    d = d[0]

    for card in cards:
      if not card in d['white_cards']:
        return self.formatted(success = False, message = 'user doesn\'t hold this card in his hand')

    d['chosen_cards'] = cards

    return self.formatted(success = True)

  def getAllChoices(self):
    # returns all choices from all users, but in shuffled order (important)
    choices = [u['chosen_cards'] for u in self.users if self.users.index(u)>0]

    random.shuffle(choices)

    return choices

  def decide(self, cards):

    # let's see to which user those cards match
    user = [u for u in self.users if u['chosen_cards'] == cards]

    if len(user) != 1:
      return self.formatted(success = False, message = "no user found with those cards")

    # the user needs to gain one point

    user = user[0]

    user['black_cards'] += 1

    # at next, all chosen cards must be removed from the user's white cards pile
    # and the chosen cards must be resetted
    for user in self.users:
      for card in user['chosen_cards']:
        del user['white_cards'][user['white_cards'].index(card)]
      user['chosen_cards'] = []

    # as next we remove the current black card
    del self.black_cards[0]

    # we distribute new white cards to all users
    # all users need to get the same amount
    for user in self.users:
      if self.users.index(user) == 0:
        continue # the czar didn't lose cards
      for i in range(len(cards)):
        if len(self.white_cards) == 0:
          # we need to load the white cards pile all new
          self.loadCards()

        user['white_cards'].append(self.white_cards[0])
        del self.white_cards[0]

    # and of course the current czar must be moved to the end of the line
    self.users.append(self.users[0])
    del self.users[0]

    if self.getCurrentBlackCard() is None:
      # the game finished and we can reset it
      end = True
      self.open = True
      self.running = False
      self.white_cards = []
      self.black_cards = []
      for user in self.users:
        user['white_cards'] = []
        user['black_cards'] = 0

      self.loadCards()

    else:
      end = False

    return self.formatted(success = True, winner = self.factory.findUser(user['user']), end = end)

  def isCreator(self, user):

    possible_users = [u for u in self.users if u['user'] == user.id]

    if len(possible_users) == 0:
      return False

    return possible_users[0]['creator']

  @staticmethod
  def userdict(user, creator):
    return {
            'user': user.id,
            'joined': True,
            'black_cards': 0,
            'white_cards': [],
            'chosen_cards': [],
            'creator': creator
           }

  @staticmethod
  def formatted(**kwargs):
    return kwargs

  @property
  def protected(self):
    return self.password_hash != None

  @property
  def id(self):
    return self.uuid.int

  @property
  def choices_remaining(self):
    return len([u for u in self.users if len(u['chosen_cards']) == 0 and self.users.index(u)>0])

  @property
  def points(self):
    return [(self.factory.findUser(u['user']), u['black_cards'], ) for u in self.users]
<commit_msg>fixed re-shuffling all black cards anew when the game should have ended<commit_after>import copy
import json
import random
import uuid

from twisted.logger import Logger

from . import version

class Game(object):
  log = Logger()

  def __init__(self, factory):
    self.factory = factory
    self.black_cards = []
    self.database_hash = None
    self.name = ''
    self.open = True
    self.password_hash = None
    self.running = False
    self.uuid = None
    self.users = []
    self.white_cards = []

  @classmethod
  def create(cls, factory, name, password_hash = None):
    game = cls(factory)
    game.database_hash = factory.card_database.hash
    game.name = name
    game.password_hash = password_hash
    game.uuid = uuid.uuid4()
    game.loadCards()
    return game

  @classmethod
  def load(cls, factory, **data):
    game = cls(factory)
    game.name = data['name']
    game.password_hash = data['password_hash'] if len(data['password_hash']) else None
    game.open = False
    game.database_hash = data['database_hash']
    game.uuid = uuid.UUID(data['id'])

    game.users = json.loads(data['users'])
    for user in game.users:
      user['joined'] = False
      user['chosen_cards'] = []
      user['white_cards'] = [factory.card_database.getCard(c) for c in user['white_cards']]

    game.white_cards = [factory.card_database.getCard(c) for c in json.loads(data['cards'])['white_cards']]
    game.black_cards = [factory.card_database.getCard(c) for c in json.loads(data['cards'])['black_cards']]

    return game

  def loadCards(self):

    if len(self.black_cards) == 0:
      self.black_cards = self.factory.card_database.getBlackCards()
      random.shuffle(self.black_cards)

    if len(self.white_cards) == 0:
      self.white_cards = self.factory.card_database.getWhiteCards()

      # we need to strip all white cards users currently have in their pile
      for user in self.users:
        for card in user['white_cards']:
          del self.white_cards[self.white_cards.index(card)]

      random.shuffle(self.white_cards)

  def mayJoin(self, user):
    if self.running:
      return self.formatted(join = False, message = 'game already running')
    if self.open:
      if len(self.users) + 1 > self.factory.card_database.max_players_per_game:
        return self.formatted(join=False, message='no more players allowed')
      return self.formatted(join=True)
    else:
      possible_users = [u for u in self.users if u['user'] == user.id]

      if len(possible_users) != 1:
        return self.formatted(join = False, message = 'you are no member of this paused game')

      if possible_users[0]['joined']:
        return self.formatted(join = False, message = 'already joined this game')

      return self.formatted(join = True)

  def join(self, user, password):
    joinable = self.mayJoin(user)
    if not joinable['join']:
      return self.formatted(success=False, message=joinable['message'])
    if self.protected and password != self.password_hash:
      return self.formatted(success=False, message='wrong password supplied')

    if self.open:
      self.users.append(self.userdict(user, len(self.users) == 0))
      user.setGame(self)
    else:
      possible_users = [u for u in self.users if u['user'] == user.id and not u['joined']]
      possible_users[0]['joined'] = True
      user.setGame(self)

    return self.formatted(success=True, game_id = self.id)

  def getAllUsers(self):
    return [self.factory.findUser(u['user']) for u in self.users if u['joined']]

  def start(self):
    if len(self.getAllUsers())<3:
      return self.formatted(success=False, message='not enough players in this game')

    if self.running:
      return self.formatted(success=False, message='already running')

    # if the game is currently open, we need to shuffle the users
    if self.open:
      random.shuffle(self.users)
      self.open = False

    self.open = False
    self.running = True

    # all users need to get 10 cards
    for i in range(len(self.users)*10):
      self.users[i/10]['white_cards'].append(self.white_cards[i])
    self.white_cards=self.white_cards[len(self.users)*10:]

    # determine the one with the black card
    # index at 0 will always be the czar
    # and black_cards 0 will always be the current black card

    return self.formatted(success=True)

  def getCurrentBlackCard(self):
    try:
      return self.black_cards[0]
    except IndexError:
      return None

  def getAllWhiteCardsForUsers(self):
    return [(self.factory.findUser(self.users[i]['user']), self.users[i]['white_cards']) for i in range(len(self.users))]

  def suspend(self, user):

    self.pause()

    if user.getGame() is not self:
      self.log.warn('user {user} not in game {game}', user = user.id, game = self.id)
      return

    possible_users = [u for u in self.users if u['user'] == user.id and u['joined']]
    if len(possible_users) != 1:
      self.log.warn('found {count} users in game {game} while suspending due to user {user}', count = len(possible_users), game = self.id, user = user.id)
    else:
      possible_users[0]['joined'] = False
      user.setGame(None)
      self.log.info('user {user} suspended game {game}', user = user.id, game = self.id)

  def leave(self, user):
    # forces the user to leave

    if user.getGame() is not self:
      return self.formatted(success = False, message = 'user is not in this game')

    if len(self.getAllUsers()) == 0:
      self.log.warn('no users in this game, {user} tried to leave', user = user.id)
      return self.formatted(success = False, message = 'no users found in this game')

    possible_users = [u for u in self.users if u['user'] == user.id and u['joined']]

    if len(possible_users) != 1:
      self.log.warn('{user} tried to leave game {game}, but found {count} possible users', user = user.id, game = self.id, count = len(possible_users))
      return self.formatted(success = False, message = 'unable to find user in this game')

    if possible_users[0]['creator']:
      for u in self.users:
        if u != possible_users[0] and not u['creator']:
          u['creator'] = True
          break

    del self.users[self.users.index(possible_users[0])]
    user.setGame(None)

    self.log.info('user {user} left game {game}', user = user.id, game = self.id)

    self.pause()

    if len(self.users)<3:
      self.open = True
      for user in self.users:
        user['white_cards'] = []
        user['chosen_cards'] = []
        user['black_cards'] = 0
      self.white_cards = []
      self.black_cards = []
      self.loadCards()

      self.log.info('not enough users left, game opened up all new')

    return self.formatted(success = True, unlinked = self.unlink())

  def pause(self):

    if not self.running:
      return

    white_cards = []

    self.running = False

    for user in self.users:
      user['chosen_cards'] = []
      white_cards += user['white_cards']
      user['white_cards'] = []

    white_cards += self.white_cards

    self.white_cards = white_cards

    self.log.info('game {game} paused', game = self.id)

  def unlink(self, force = False):
    # can only unlink if no users are left
    # as long as it is not forced (force by using the delete button)
    if len(self.users)>0 and not force:
      return False

    self.factory.unlinkGame(self)
    self.log.info('game {game} deleted', game = self.id)
    return True

  def pack(self):
    # saves the current game into the database
    # at first we construct the sql arguments dict

    args = {}

    args['id'] = self.uuid.hex
    args['name'] = self.name

    users = copy.deepcopy(self.users)
    for user in users:
      del user['joined']
      del user['chosen_cards']
      user['white_cards'] = [c.id for c in user['white_cards']]

    args['users'] = json.dumps(users)

    black_cards = [c.id for c in self.black_cards]
    white_cards = [c.id for c in self.white_cards]

    args['cards'] = json.dumps({'white_cards': white_cards, 'black_cards': black_cards})

    args['password_hash'] = self.password_hash if self.protected else ''
    args['database_hash'] = self.database_hash
    args['server_version_major'] = version.MAJOR
    args['server_version_minor'] = version.MINOR
    args['server_version_revision'] = version.REVISION

    return args

  def chooseCards(self, user, cards):
    d = [u for u in self.users if u['user'] == user.id]

    if len(d) != 1:
      self.log.warn('user {user} wants to choose cards, but user not found in game {game}', user = user.id, game = self.id)
      return self.formatted(success = False, message = 'user not in this game')

    d = d[0]

    for card in cards:
      if not card in d['white_cards']:
        return self.formatted(success = False, message = 'user doesn\'t hold this card in his hand')

    d['chosen_cards'] = cards

    return self.formatted(success = True)

  def getAllChoices(self):
    # returns all choices from all users, but in shuffled order (important)
    choices = [u['chosen_cards'] for u in self.users if self.users.index(u)>0]

    random.shuffle(choices)

    return choices

  def decide(self, cards):

    # let's see to which user those cards match
    user = [u for u in self.users if u['chosen_cards'] == cards]

    if len(user) != 1:
      return self.formatted(success = False, message = "no user found with those cards")

    # the user needs to gain one point

    user = user[0]

    user['black_cards'] += 1

    # as next we remove the current black card
    del self.black_cards[0]

    next_black_card = self.getCurrentBlackCard()

    # at next, all chosen cards must be removed from the user's white cards pile
    # and the chosen cards must be resetted
    for user in self.users:
      for card in user['chosen_cards']:
        del user['white_cards'][user['white_cards'].index(card)]
      user['chosen_cards'] = []

    # we distribute new white cards to all users
    # all users need to get the same amount
    for user in self.users:
      if self.users.index(user) == 0:
        continue # the czar didn't lose cards
      for i in range(len(cards)):
        if len(self.white_cards) == 0:
          # we need to load the white cards pile all new
          self.loadCards()

        user['white_cards'].append(self.white_cards[0])
        del self.white_cards[0]

    # and of course the current czar must be moved to the end of the line
    self.users.append(self.users[0])
    del self.users[0]

    if next_black_card is None:
      # the game finished and we can reset it
      end = True
      self.open = True
      self.running = False
      self.white_cards = []
      self.black_cards = []
      for user in self.users:
        user['white_cards'] = []
        user['black_cards'] = 0

      self.loadCards()

    else:
      end = False

    return self.formatted(success = True, winner = self.factory.findUser(user['user']), end = end)

  def isCreator(self, user):

    possible_users = [u for u in self.users if u['user'] == user.id]

    if len(possible_users) == 0:
      return False

    return possible_users[0]['creator']

  @staticmethod
  def userdict(user, creator):
    return {
            'user': user.id,
            'joined': True,
            'black_cards': 0,
            'white_cards': [],
            'chosen_cards': [],
            'creator': creator
           }

  @staticmethod
  def formatted(**kwargs):
    return kwargs

  @property
  def protected(self):
    return self.password_hash != None

  @property
  def id(self):
    return self.uuid.int

  @property
  def choices_remaining(self):
    return len([u for u in self.users if len(u['chosen_cards']) == 0 and self.users.index(u)>0])

  @property
  def points(self):
    return [(self.factory.findUser(u['user']), u['black_cards'], ) for u in self.users]
<|endoftext|>


<commit_before>import time
from distutils.version import LooseVersion

from flaky import flaky

from dtest import CASSANDRA_VERSION_FROM_BUILD, Tester, debug
from thrift_bindings.v22.ttypes import \
    ConsistencyLevel as ThriftConsistencyLevel
from thrift_bindings.v22.ttypes import (CfDef, Column, ColumnOrSuperColumn,
                                        ColumnParent, CounterColumn, KsDef,
                                        Mutation, SlicePredicate, SliceRange,
                                        SuperColumn, TimedOutException)
from thrift_tests import get_thrift_client
from tools import RerunTestException, requires_rerun, since, known_failure

@since('2.0', max_version='2.1.x')
class TestSCUpgrade(Tester):
    """
    Tests upgrade between 1.2->2.0 for super columns (since that's where we
    removed then internally).
    """

    def __init__(self, *args, **kwargs):
        self.ignore_log_patterns = [
            # This one occurs if we do a non-rolling upgrade, the node
            # it's trying to send the migration to hasn't started yet,
            # and when it does, it gets replayed and everything is fine.
            r'Can\'t send migration request: node.*is down',
        ]
        if LooseVersion(CASSANDRA_VERSION_FROM_BUILD) < '2.2':
            _known_teardown_race_error = (
                'ScheduledThreadPoolExecutor$ScheduledFutureTask@[0-9a-f]+ '
                'rejected from org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor'
            )
            # don't alter ignore_log_patterns on the class, just the obj for this test
            self.ignore_log_patterns += [_known_teardown_race_error]

        Tester.__init__(self, *args, **kwargs)

    def upgrade_with_index_creation_test(self):
        cluster = self.cluster

        # Forcing cluster version on purpose
        cluster.set_install_dir(version="1.2.16")
        if "memtable_allocation_type" in cluster._config_options:
            cluster._config_options.__delitem__("memtable_allocation_type")
        cluster.populate(2).start()

        node1, node2 = cluster.nodelist()

        # wait for the rpc server to start
        session = self.patient_exclusive_cql_connection(node1)

        host, port = node1.network_interfaces['thrift']
        client = get_thrift_client(host, port)
        client.transport.open()

        ksdef = KsDef()
        ksdef.name = 'test'
        ksdef.strategy_class = 'SimpleStrategy'
        ksdef.strategy_options = {'replication_factor': '2'}
        ksdef.durable_writes = True
        ksdef.cf_defs = []

        client.system_add_keyspace(ksdef)

        session.cluster.control_connection.wait_for_schema_agreement()

        client.set_keyspace('test')

        # create a super column family with UTF8 for all types
        cfdef = CfDef()
        cfdef.keyspace = 'test'
        cfdef.name = 'sc_test'
        cfdef.column_type = 'Super'
        cfdef.comparator_type = 'UTF8Type'
        cfdef.subcomparator_type = 'UTF8Type'
        cfdef.key_validation_class = 'UTF8Type'
        cfdef.default_validation_class = 'UTF8Type'
        cfdef.caching = 'rows_only'

        client.system_add_column_family(cfdef)

        session.cluster.control_connection.wait_for_schema_agreement()

        for i in range(2):
            supercol_name = 'sc%d' % i
            for j in range(2):
                col_name = 'c%d' % j
                column = Column(name=col_name, value='v', timestamp=100)
                client.batch_mutate(
                    {'k0': {'sc_test': [Mutation(ColumnOrSuperColumn(super_column=SuperColumn(supercol_name, [column])))]}},
                    ThriftConsistencyLevel.ONE)

        session.cluster.shutdown()
        client.transport.close()

        # Upgrade nodes to 2.0 for intermediate sstable conversion
        # See CASSANDRA-7008
        self.upgrade_to_version("binary:2.0.17")
        time.sleep(.5)

        # Upgrade node 1
        node1.flush()
        time.sleep(.5)
        node1.stop(wait_other_notice=True)
        self.set_node_to_current_version(node1)
        node1.start(wait_other_notice=True, wait_for_binary_proto=True)
        time.sleep(.5)

        # wait for the RPC server to start
        session = self.patient_exclusive_cql_connection(node1)

        client = get_thrift_client(host, port)
        client.transport.open()
        client.set_keyspace('test')

        # fetch all supercolumns
        column_parent = ColumnParent(column_family='sc_test')
        predicate = SlicePredicate(slice_range=SliceRange("", "", False, 100))
        super_columns = client.get_slice('k0', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(super_columns))
        for i in range(2):
            super_column = super_columns[i].super_column
            self.assertEqual('sc%d' % i, super_column.name)
            self.assertEqual(2, len(super_column.columns))
            for j in range(2):
                column = super_column.columns[j]
                self.assertEqual('c%d' % j, column.name)
                self.assertEqual('v', column.value)

        # fetch a single supercolumn
        column_parent = ColumnParent(column_family='sc_test', super_column='sc1')
        columns = client.get_slice('k0', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(columns))
        for j in range(2):
            column = columns[j].column
            self.assertEqual('c%d' % j, column.name)
            self.assertEqual('v', column.value)

        # fetch a single subcolumn
        predicate = SlicePredicate(column_names=['c1'])
        columns = client.get_slice('k0', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(1, len(columns))
        column = columns[0].column
        self.assertEqual('c%d' % j, column.name)
        self.assertEqual('v', column.value)

    @known_failure(failure_source='test',
                   jira_url='https://issues.apache.org/jira/browse/CASSANDRA-12340',
                   flaky=True)
    @flaky(max_runs=3, rerun_filter=requires_rerun)
    def upgrade_with_counters_test(self):
        cluster = self.cluster

        # Forcing cluster version on purpose
        cluster.set_install_dir(version="1.2.19")
        if "memtable_allocation_type" in cluster._config_options:
            cluster._config_options.__delitem__("memtable_allocation_type")
        cluster.populate(3).start()

        node1, node2, node3 = cluster.nodelist()

        # wait for the rpc server to start
        session = self.patient_exclusive_cql_connection(node1)

        host, port = node1.network_interfaces['thrift']
        client = get_thrift_client(host, port)
        client.transport.open()

        ksdef = KsDef()
        ksdef.name = 'test'
        ksdef.strategy_class = 'SimpleStrategy'
        ksdef.strategy_options = {'replication_factor': '2'}
        ksdef.durable_writes = True
        ksdef.cf_defs = []

        client.system_add_keyspace(ksdef)
        client.set_keyspace('test')

        # create a super column family with UTF8 for all types except for the
        # values, which are counters
        cfdef = CfDef()
        cfdef.keyspace = 'test'
        cfdef.name = 'sc_test'
        cfdef.column_type = 'Super'
        cfdef.comparator_type = 'UTF8Type'
        cfdef.subcomparator_type = 'UTF8Type'
        cfdef.key_validation_class = 'UTF8Type'
        cfdef.default_validation_class = 'CounterColumnType'

        client.system_add_column_family(cfdef)

        session.cluster.control_connection.wait_for_schema_agreement()

        for i in range(2):
            supercol_name = 'sc%d' % i
            column_parent = ColumnParent(column_family='sc_test', super_column=supercol_name)
            for j in range(2):
                col_name = 'c%d' % j
                column = CounterColumn(name=col_name, value=1)
                for k in range(20):
                    try:
                        client.add('Counter1', column_parent, column, ThriftConsistencyLevel.ONE)
                    except TimedOutException:
                        raise RerunTestException("re-run test to verify")

        # If we are on 2.1 or any higher version upgrade to 2.0.latest.
        # Otherwise, we must be on a 2.0.x, so we should be upgrading to that version.
        # This will let us test upgrading from 1.2.19 to each of the 2.0 minor releases.
        # Upgrade nodes to 2.0.
        # See CASSANDRA-7008
        self.upgrade_to_version("binary:2.0.17", [node1])
        time.sleep(.5)

        # wait for the RPC server to start
        session = self.patient_exclusive_cql_connection(node1)

        for node in (node1, node2, node3):
            host, port = node.network_interfaces['thrift']
            client = get_thrift_client(host, port)
            client.transport.open()
            client.set_keyspace('test')
            for i in range(2):
                supercol_name = 'sc%d' % i
                column_parent = ColumnParent(column_family='sc_test', super_column=supercol_name)
                for j in range(2):
                    col_name = 'c%d' % j
                    column = CounterColumn(name=col_name, value=1)
                    for k in range(50):
                        try:
                            client.add('Counter1', column_parent, column, ThriftConsistencyLevel.ONE)
                        except TimedOutException:
                            raise RerunTestException("re-run test to verify")

            client.transport.close()

        # Upgrade nodes to 2.0.
        # See CASSANDRA-7008
        self.upgrade_to_version("binary:2.0.17", [node2, node3])
        time.sleep(.5)

        host, port = node1.network_interfaces['thrift']
        client = get_thrift_client(host, port)
        client.transport.open()
        client.set_keyspace('test')

        column_parent = ColumnParent(column_family='sc_test')
        predicate = SlicePredicate(slice_range=SliceRange("", "", False, 100))
        super_columns = client.get_slice('Counter1', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(super_columns))
        for i in range(2):
            super_column = super_columns[i].counter_super_column
            self.assertEqual('sc%d' % i, super_column.name)
            self.assertEqual(2, len(super_column.columns))
            for j in range(2):
                column = super_column.columns[j]
                self.assertEqual('c%d' % j, column.name)
                self.assertEqual(170, column.value)

        # fetch a single supercolumn
        column_parent = ColumnParent(column_family='sc_test', super_column='sc1')
        columns = client.get_slice('Counter1', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(columns))
        for j in range(2):
            column = columns[j].counter_column
            self.assertEqual('c%d' % j, column.name)
            self.assertEqual(170, column.value)

        # fetch a single subcolumn
        predicate = SlicePredicate(column_names=['c1'])
        columns = client.get_slice('Counter1', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(1, len(columns))
        column = columns[0].counter_column
        self.assertEqual('c%d' % j, column.name)
        self.assertEqual(170, column.value)

    def upgrade_to_version(self, tag, nodes=None):
        debug('Upgrading to ' + tag)
        if nodes is None:
            nodes = self.cluster.nodelist()

        for node in nodes:
            debug('Shutting down node: ' + node.name)
            node.drain()
            node.watch_log_for("DRAINED")
            node.stop(wait_other_notice=False)

        # Update Cassandra Directory
        for node in nodes:
            node.set_install_dir(version=tag)
            if tag < "2.1":
                if "memtable_allocation_type" in node.config_options:
                    node.config_options.__delitem__("memtable_allocation_type")
            debug("Set new cassandra dir for %s: %s" % (node.name, node.get_install_dir()))
        self.cluster.set_install_dir(version=tag)

        # Restart nodes on new version
        for node in nodes:
            debug('Starting %s on new version (%s)' % (node.name, tag))
            # Setup log4j / logback again (necessary moving from 2.0 -> 2.1):
            node.set_log_level("INFO")
            node.start(wait_other_notice=True, wait_for_binary_proto=True)
            node.nodetool('upgradesstables -a')
<commit_msg>isort upgrade_supercolumns_test<commit_after>import time
from distutils.version import LooseVersion

from flaky import flaky

from dtest import CASSANDRA_VERSION_FROM_BUILD, Tester, debug
from thrift_bindings.v22.ttypes import \
    ConsistencyLevel as ThriftConsistencyLevel
from thrift_bindings.v22.ttypes import (CfDef, Column, ColumnOrSuperColumn,
                                        ColumnParent, CounterColumn, KsDef,
                                        Mutation, SlicePredicate, SliceRange,
                                        SuperColumn, TimedOutException)
from thrift_tests import get_thrift_client
from tools import RerunTestException, known_failure, requires_rerun, since

@since('2.0', max_version='2.1.x')
class TestSCUpgrade(Tester):
    """
    Tests upgrade between 1.2->2.0 for super columns (since that's where we
    removed then internally).
    """

    def __init__(self, *args, **kwargs):
        self.ignore_log_patterns = [
            # This one occurs if we do a non-rolling upgrade, the node
            # it's trying to send the migration to hasn't started yet,
            # and when it does, it gets replayed and everything is fine.
            r'Can\'t send migration request: node.*is down',
        ]
        if LooseVersion(CASSANDRA_VERSION_FROM_BUILD) < '2.2':
            _known_teardown_race_error = (
                'ScheduledThreadPoolExecutor$ScheduledFutureTask@[0-9a-f]+ '
                'rejected from org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor'
            )
            # don't alter ignore_log_patterns on the class, just the obj for this test
            self.ignore_log_patterns += [_known_teardown_race_error]

        Tester.__init__(self, *args, **kwargs)

    def upgrade_with_index_creation_test(self):
        cluster = self.cluster

        # Forcing cluster version on purpose
        cluster.set_install_dir(version="1.2.16")
        if "memtable_allocation_type" in cluster._config_options:
            cluster._config_options.__delitem__("memtable_allocation_type")
        cluster.populate(2).start()

        node1, node2 = cluster.nodelist()

        # wait for the rpc server to start
        session = self.patient_exclusive_cql_connection(node1)

        host, port = node1.network_interfaces['thrift']
        client = get_thrift_client(host, port)
        client.transport.open()

        ksdef = KsDef()
        ksdef.name = 'test'
        ksdef.strategy_class = 'SimpleStrategy'
        ksdef.strategy_options = {'replication_factor': '2'}
        ksdef.durable_writes = True
        ksdef.cf_defs = []

        client.system_add_keyspace(ksdef)

        session.cluster.control_connection.wait_for_schema_agreement()

        client.set_keyspace('test')

        # create a super column family with UTF8 for all types
        cfdef = CfDef()
        cfdef.keyspace = 'test'
        cfdef.name = 'sc_test'
        cfdef.column_type = 'Super'
        cfdef.comparator_type = 'UTF8Type'
        cfdef.subcomparator_type = 'UTF8Type'
        cfdef.key_validation_class = 'UTF8Type'
        cfdef.default_validation_class = 'UTF8Type'
        cfdef.caching = 'rows_only'

        client.system_add_column_family(cfdef)

        session.cluster.control_connection.wait_for_schema_agreement()

        for i in range(2):
            supercol_name = 'sc%d' % i
            for j in range(2):
                col_name = 'c%d' % j
                column = Column(name=col_name, value='v', timestamp=100)
                client.batch_mutate(
                    {'k0': {'sc_test': [Mutation(ColumnOrSuperColumn(super_column=SuperColumn(supercol_name, [column])))]}},
                    ThriftConsistencyLevel.ONE)

        session.cluster.shutdown()
        client.transport.close()

        # Upgrade nodes to 2.0 for intermediate sstable conversion
        # See CASSANDRA-7008
        self.upgrade_to_version("binary:2.0.17")
        time.sleep(.5)

        # Upgrade node 1
        node1.flush()
        time.sleep(.5)
        node1.stop(wait_other_notice=True)
        self.set_node_to_current_version(node1)
        node1.start(wait_other_notice=True, wait_for_binary_proto=True)
        time.sleep(.5)

        # wait for the RPC server to start
        session = self.patient_exclusive_cql_connection(node1)

        client = get_thrift_client(host, port)
        client.transport.open()
        client.set_keyspace('test')

        # fetch all supercolumns
        column_parent = ColumnParent(column_family='sc_test')
        predicate = SlicePredicate(slice_range=SliceRange("", "", False, 100))
        super_columns = client.get_slice('k0', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(super_columns))
        for i in range(2):
            super_column = super_columns[i].super_column
            self.assertEqual('sc%d' % i, super_column.name)
            self.assertEqual(2, len(super_column.columns))
            for j in range(2):
                column = super_column.columns[j]
                self.assertEqual('c%d' % j, column.name)
                self.assertEqual('v', column.value)

        # fetch a single supercolumn
        column_parent = ColumnParent(column_family='sc_test', super_column='sc1')
        columns = client.get_slice('k0', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(columns))
        for j in range(2):
            column = columns[j].column
            self.assertEqual('c%d' % j, column.name)
            self.assertEqual('v', column.value)

        # fetch a single subcolumn
        predicate = SlicePredicate(column_names=['c1'])
        columns = client.get_slice('k0', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(1, len(columns))
        column = columns[0].column
        self.assertEqual('c%d' % j, column.name)
        self.assertEqual('v', column.value)

    @known_failure(failure_source='test',
                   jira_url='https://issues.apache.org/jira/browse/CASSANDRA-12340',
                   flaky=True)
    @flaky(max_runs=3, rerun_filter=requires_rerun)
    def upgrade_with_counters_test(self):
        cluster = self.cluster

        # Forcing cluster version on purpose
        cluster.set_install_dir(version="1.2.19")
        if "memtable_allocation_type" in cluster._config_options:
            cluster._config_options.__delitem__("memtable_allocation_type")
        cluster.populate(3).start()

        node1, node2, node3 = cluster.nodelist()

        # wait for the rpc server to start
        session = self.patient_exclusive_cql_connection(node1)

        host, port = node1.network_interfaces['thrift']
        client = get_thrift_client(host, port)
        client.transport.open()

        ksdef = KsDef()
        ksdef.name = 'test'
        ksdef.strategy_class = 'SimpleStrategy'
        ksdef.strategy_options = {'replication_factor': '2'}
        ksdef.durable_writes = True
        ksdef.cf_defs = []

        client.system_add_keyspace(ksdef)
        client.set_keyspace('test')

        # create a super column family with UTF8 for all types except for the
        # values, which are counters
        cfdef = CfDef()
        cfdef.keyspace = 'test'
        cfdef.name = 'sc_test'
        cfdef.column_type = 'Super'
        cfdef.comparator_type = 'UTF8Type'
        cfdef.subcomparator_type = 'UTF8Type'
        cfdef.key_validation_class = 'UTF8Type'
        cfdef.default_validation_class = 'CounterColumnType'

        client.system_add_column_family(cfdef)

        session.cluster.control_connection.wait_for_schema_agreement()

        for i in range(2):
            supercol_name = 'sc%d' % i
            column_parent = ColumnParent(column_family='sc_test', super_column=supercol_name)
            for j in range(2):
                col_name = 'c%d' % j
                column = CounterColumn(name=col_name, value=1)
                for k in range(20):
                    try:
                        client.add('Counter1', column_parent, column, ThriftConsistencyLevel.ONE)
                    except TimedOutException:
                        raise RerunTestException("re-run test to verify")

        # If we are on 2.1 or any higher version upgrade to 2.0.latest.
        # Otherwise, we must be on a 2.0.x, so we should be upgrading to that version.
        # This will let us test upgrading from 1.2.19 to each of the 2.0 minor releases.
        # Upgrade nodes to 2.0.
        # See CASSANDRA-7008
        self.upgrade_to_version("binary:2.0.17", [node1])
        time.sleep(.5)

        # wait for the RPC server to start
        session = self.patient_exclusive_cql_connection(node1)

        for node in (node1, node2, node3):
            host, port = node.network_interfaces['thrift']
            client = get_thrift_client(host, port)
            client.transport.open()
            client.set_keyspace('test')
            for i in range(2):
                supercol_name = 'sc%d' % i
                column_parent = ColumnParent(column_family='sc_test', super_column=supercol_name)
                for j in range(2):
                    col_name = 'c%d' % j
                    column = CounterColumn(name=col_name, value=1)
                    for k in range(50):
                        try:
                            client.add('Counter1', column_parent, column, ThriftConsistencyLevel.ONE)
                        except TimedOutException:
                            raise RerunTestException("re-run test to verify")

            client.transport.close()

        # Upgrade nodes to 2.0.
        # See CASSANDRA-7008
        self.upgrade_to_version("binary:2.0.17", [node2, node3])
        time.sleep(.5)

        host, port = node1.network_interfaces['thrift']
        client = get_thrift_client(host, port)
        client.transport.open()
        client.set_keyspace('test')

        column_parent = ColumnParent(column_family='sc_test')
        predicate = SlicePredicate(slice_range=SliceRange("", "", False, 100))
        super_columns = client.get_slice('Counter1', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(super_columns))
        for i in range(2):
            super_column = super_columns[i].counter_super_column
            self.assertEqual('sc%d' % i, super_column.name)
            self.assertEqual(2, len(super_column.columns))
            for j in range(2):
                column = super_column.columns[j]
                self.assertEqual('c%d' % j, column.name)
                self.assertEqual(170, column.value)

        # fetch a single supercolumn
        column_parent = ColumnParent(column_family='sc_test', super_column='sc1')
        columns = client.get_slice('Counter1', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(2, len(columns))
        for j in range(2):
            column = columns[j].counter_column
            self.assertEqual('c%d' % j, column.name)
            self.assertEqual(170, column.value)

        # fetch a single subcolumn
        predicate = SlicePredicate(column_names=['c1'])
        columns = client.get_slice('Counter1', column_parent, predicate, ThriftConsistencyLevel.QUORUM)
        self.assertEqual(1, len(columns))
        column = columns[0].counter_column
        self.assertEqual('c%d' % j, column.name)
        self.assertEqual(170, column.value)

    def upgrade_to_version(self, tag, nodes=None):
        debug('Upgrading to ' + tag)
        if nodes is None:
            nodes = self.cluster.nodelist()

        for node in nodes:
            debug('Shutting down node: ' + node.name)
            node.drain()
            node.watch_log_for("DRAINED")
            node.stop(wait_other_notice=False)

        # Update Cassandra Directory
        for node in nodes:
            node.set_install_dir(version=tag)
            if tag < "2.1":
                if "memtable_allocation_type" in node.config_options:
                    node.config_options.__delitem__("memtable_allocation_type")
            debug("Set new cassandra dir for %s: %s" % (node.name, node.get_install_dir()))
        self.cluster.set_install_dir(version=tag)

        # Restart nodes on new version
        for node in nodes:
            debug('Starting %s on new version (%s)' % (node.name, tag))
            # Setup log4j / logback again (necessary moving from 2.0 -> 2.1):
            node.set_log_level("INFO")
            node.start(wait_other_notice=True, wait_for_binary_proto=True)
            node.nodetool('upgradesstables -a')
<|endoftext|>