<commit_before>class DictOutput():

  def __init__(self, mode='CamelCase'):
    self.map = dict()
    self.coord_map = ["left", "top", "right", "bottom"]

    if mode == 'CamelCase':
        self.fillDictCamelCase()
    else:
        self.fillDictOld()

  def fillDictCamelCase(self):
    self.map['query_sha1'] = "QuerySha1"
    self.map['query_url'] = "QueryURL"
    self.map['query_face'] = "QueryFace"
    self.map['image_sha1s'] = "ImageSha1s"
    self.map['similar_faces'] = "SimilarFaces"
    self.map['similar_images'] = "SimilarImages"
    self.map['faces'] = "Faces"
    self.map['img_info'] = "ImgInfo"
    self.map['distances'] = "Distances"
    self.map['number_images'] = "NumberImages"
    self.map['number_faces'] = "NumberFaces"
    self.map['number_similar_faces'] = "NumberSimilarFaces"
    self.map['number_similar_images'] = "NumberSimilarImages"
    self.map['all_similar_faces'] = "AllSimilarFaces"
    self.map['all_similar_images'] = "AllSimilarImages"
    self.map['cached_image_urls'] = "CachedImageURLs"

  def fillDictOld(self):
    self.map['query_sha1'] = "query_sha1"
    self.map['query_url'] = "query_url"
    self.map['query_face'] = "query_face"
    self.map['image_sha1s'] = "image_sha1s"
    self.map['similar_faces'] = "similar_faces"
    self.map['similar_images'] = "similar_images"
    self.map['faces'] = "faces"
    self.map['img_info'] = "img_info"
    self.map['distances'] = "distances"
    self.map['number_images'] = "number_images"
    self.map['number_faces'] = "number_faces"
    self.map['number_similar_faces'] = "number_similar_faces"
    self.map['number_similar_images'] = "number_similar_images"
    self.map['all_similar_faces'] = "all_similar_faces"
    self.map['all_similar_images'] = "all_similar_images"
    self.map['cached_image_urls'] = "cached_image_urls"

  def format_output(self, dets, sim_images, sim_dets, sim_score, options_dict=dict(), input_type="image"):
    import time
    from collections import OrderedDict
    print "[format_output: log] options are: {}".format(options_dict)
    start_build_output = time.time()
    output = []
    images_query = set()
    nb_faces_query = 0
    nb_faces_similar = 0
    nb_images_similar = 0

    #print 'dets',len(dets),dets
    #print 'sim_dets', len(sim_dets), sim_dets

    if input_type != "image":

      for i in range(len(dets)):

        # No detection in query
        if not dets[i][1]:
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:5]
          images_query.add(dets[i][0])
          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], 0],
                                                                 [self.map['image_sha1s'], []],
                                                                 [self.map[input_type+'s'], []],
                                                                 [self.map['cached_image_urls'], []],
                                                                 [self.map['distances'], []]])
          continue

        # We found some faces...
        #print len(dets[i][1])
        for j, face_bbox in enumerate(dets[i][1]):
          nb_faces_query += 1

          # Add one output for each face query
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          output[out_i][self.map['query_'+input_type]] = dets[i][1][j]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:]
          images_query.add(dets[i][0])

          nb_dets = 0
          #print "sim_dets[i]",sim_dets[i]

          if sim_dets[i]:
            if len(sim_dets[i])>j and sim_dets[i][j]:
              #print "sim_dets[i][j]",sim_dets[i][j]
              nb_dets = len(sim_dets[i][j])

          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], nb_faces],
                                                                [self.map['image_sha1s'], []],
                                                                [self.map[input_type+'s'], []],
                                                                [self.map['img_info'], []],
                                                                [self.map['cached_image_urls'], []],
                                                                [self.map['distances'], []]])

          #print 'nb_faces: %d' % nb_faces

          # Explore list of similar faces
          for jj in range(nb_dets):
            sim_det = sim_dets[i][j][jj]
            nb_faces_similar += 1
            output[out_i][self.map['similar_'+input_type+'s']][self.map['image_sha1s']].append(sim_images[i][j][jj][0].strip())
            output[out_i][self.map['similar_'+input_type+'s']][self.map['cached_image_urls']].append(sim_images[i][j][jj][1][self.url_field].strip())

            tmp_face_dict = dict()
            for tfi, tfcoord in enumerate(sim_det.split('_')[1:]):
              tmp_face_dict[self.coord_map[tfi]] = int(tfcoord)
            output[out_i][self.map['similar_'+input_type+'s']][self.map[input_type+'s']].append(tmp_face_dict)
            # this is not in HBase for all/most images...
            #osf_imginfo = sim_images[i][j][jj][1][self.img_info_field].strip()
            output[out_i][self.map['similar_'+input_type+'s']][self.map['img_info']].append('')
            output[out_i][self.map['similar_'+input_type+'s']][self.map['distances']].append(sim_score[i][j][jj])

      outp = OrderedDict([[self.map['number_images'], len(images_query)],
                          [self.map['number_'+input_type+'s'], nb_faces_query], # this would overwrite number_images if input_type is image...
                          [self.map['number_similar_'+input_type+'s'], nb_faces_similar],
                          [self.map['all_similar_'+input_type+'s'], output]])

    else:
      # no detections, dets contains list of query images sha1

      for i in range(len(dets)):
        sha1 = dets[i][0]

        # Add one output for each image query
        output.append(dict())
        out_i = len(output) - 1
        output[out_i][self.map['query_sha1']] = sha1
        if dets[i][1]:
          output[out_i][self.map['query_url']] = dets[i][1]

        if sim_images[i]:
          nb_images = len(sim_images[i])

        output[out_i][self.map['similar_images']] = OrderedDict([[self.map['number_images'], nb_images],
                                                              [self.map['image_sha1s'], []],
                                                              [self.map['cached_image_urls'], []],
                                                              [self.map['distances'], []]])

        # Explore list of similar faces
        for j in range(nb_images):
          print "sim_images[i][j]",sim_images[i][j]
          nb_images_similar += 1
          output[out_i][self.map['similar_images']][self.map['image_sha1s']].append(sim_images[i][j][0][0].strip())
          output[out_i][self.map['similar_images']][self.map['cached_image_urls']].append(sim_images[i][j][0][1][self.url_field].strip())
          output[out_i][self.map['similar_images']][self.map['distances']].append(sim_score[i][j][0])

      outp = OrderedDict([[self.map['number_images'], len(dets)],
                          [self.map['number_similar_images'], nb_images_similar],
                          [self.map['all_similar_images'], output]])

    print "[format_output: log] build_output took: {}".format(time.time() - start_build_output)
    return outp
<commit_msg>sim_images[i] is a list of only one list<commit_after>class DictOutput():

  def __init__(self, mode='CamelCase'):
    self.map = dict()
    self.coord_map = ["left", "top", "right", "bottom"]

    if mode == 'CamelCase':
        self.fillDictCamelCase()
    else:
        self.fillDictOld()

  def fillDictCamelCase(self):
    self.map['query_sha1'] = "QuerySha1"
    self.map['query_url'] = "QueryURL"
    self.map['query_face'] = "QueryFace"
    self.map['image_sha1s'] = "ImageSha1s"
    self.map['similar_faces'] = "SimilarFaces"
    self.map['similar_images'] = "SimilarImages"
    self.map['faces'] = "Faces"
    self.map['img_info'] = "ImgInfo"
    self.map['distances'] = "Distances"
    self.map['number_images'] = "NumberImages"
    self.map['number_faces'] = "NumberFaces"
    self.map['number_similar_faces'] = "NumberSimilarFaces"
    self.map['number_similar_images'] = "NumberSimilarImages"
    self.map['all_similar_faces'] = "AllSimilarFaces"
    self.map['all_similar_images'] = "AllSimilarImages"
    self.map['cached_image_urls'] = "CachedImageURLs"

  def fillDictOld(self):
    self.map['query_sha1'] = "query_sha1"
    self.map['query_url'] = "query_url"
    self.map['query_face'] = "query_face"
    self.map['image_sha1s'] = "image_sha1s"
    self.map['similar_faces'] = "similar_faces"
    self.map['similar_images'] = "similar_images"
    self.map['faces'] = "faces"
    self.map['img_info'] = "img_info"
    self.map['distances'] = "distances"
    self.map['number_images'] = "number_images"
    self.map['number_faces'] = "number_faces"
    self.map['number_similar_faces'] = "number_similar_faces"
    self.map['number_similar_images'] = "number_similar_images"
    self.map['all_similar_faces'] = "all_similar_faces"
    self.map['all_similar_images'] = "all_similar_images"
    self.map['cached_image_urls'] = "cached_image_urls"

  def format_output(self, dets, sim_images, sim_dets, sim_score, options_dict=dict(), input_type="image"):
    import time
    from collections import OrderedDict
    print "[format_output: log] options are: {}".format(options_dict)
    start_build_output = time.time()
    output = []
    images_query = set()
    nb_faces_query = 0
    nb_faces_similar = 0
    nb_images_similar = 0

    #print 'dets',len(dets),dets
    #print 'sim_dets', len(sim_dets), sim_dets

    if input_type != "image":

      for i in range(len(dets)):

        # No detection in query
        if not dets[i][1]:
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:5]
          images_query.add(dets[i][0])
          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], 0],
                                                                 [self.map['image_sha1s'], []],
                                                                 [self.map[input_type+'s'], []],
                                                                 [self.map['cached_image_urls'], []],
                                                                 [self.map['distances'], []]])
          continue

        # We found some faces...
        #print len(dets[i][1])
        for j, face_bbox in enumerate(dets[i][1]):
          nb_faces_query += 1

          # Add one output for each face query
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          output[out_i][self.map['query_'+input_type]] = dets[i][1][j]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:]
          images_query.add(dets[i][0])

          nb_dets = 0
          #print "sim_dets[i]",sim_dets[i]

          if sim_dets[i]:
            if len(sim_dets[i])>j and sim_dets[i][j]:
              #print "sim_dets[i][j]",sim_dets[i][j]
              nb_dets = len(sim_dets[i][j])

          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], nb_faces],
                                                                [self.map['image_sha1s'], []],
                                                                [self.map[input_type+'s'], []],
                                                                [self.map['img_info'], []],
                                                                [self.map['cached_image_urls'], []],
                                                                [self.map['distances'], []]])

          #print 'nb_faces: %d' % nb_faces

          # Explore list of similar faces
          for jj in range(nb_dets):
            sim_det = sim_dets[i][j][jj]
            nb_faces_similar += 1
            output[out_i][self.map['similar_'+input_type+'s']][self.map['image_sha1s']].append(sim_images[i][j][jj][0].strip())
            output[out_i][self.map['similar_'+input_type+'s']][self.map['cached_image_urls']].append(sim_images[i][j][jj][1][self.url_field].strip())

            tmp_face_dict = dict()
            for tfi, tfcoord in enumerate(sim_det.split('_')[1:]):
              tmp_face_dict[self.coord_map[tfi]] = int(tfcoord)
            output[out_i][self.map['similar_'+input_type+'s']][self.map[input_type+'s']].append(tmp_face_dict)
            # this is not in HBase for all/most images...
            #osf_imginfo = sim_images[i][j][jj][1][self.img_info_field].strip()
            output[out_i][self.map['similar_'+input_type+'s']][self.map['img_info']].append('')
            output[out_i][self.map['similar_'+input_type+'s']][self.map['distances']].append(sim_score[i][j][jj])

      outp = OrderedDict([[self.map['number_images'], len(images_query)],
                          [self.map['number_'+input_type+'s'], nb_faces_query], # this would overwrite number_images if input_type is image...
                          [self.map['number_similar_'+input_type+'s'], nb_faces_similar],
                          [self.map['all_similar_'+input_type+'s'], output]])

    else:
      # no detections, dets contains list of query images sha1

      for i in range(len(dets)):
        sha1 = dets[i][0]

        # Add one output for each image query
        output.append(dict())
        out_i = len(output) - 1
        output[out_i][self.map['query_sha1']] = sha1
        if dets[i][1]:
          output[out_i][self.map['query_url']] = dets[i][1]

        if sim_images[i][0]:
          nb_images = len(sim_images[i][0])

        output[out_i][self.map['similar_images']] = OrderedDict([[self.map['number_images'], nb_images],
                                                              [self.map['image_sha1s'], []],
                                                              [self.map['cached_image_urls'], []],
                                                              [self.map['distances'], []]])

        # Explore list of similar faces
        for j in range(nb_images):
          print "sim_images[i][j]",sim_images[i][0][j]
          nb_images_similar += 1
          output[out_i][self.map['similar_images']][self.map['image_sha1s']].append(sim_images[i][0][j][0].strip())
          output[out_i][self.map['similar_images']][self.map['cached_image_urls']].append(sim_images[i][0][j][1][self.url_field].strip())
          output[out_i][self.map['similar_images']][self.map['distances']].append(sim_score[i][0][j])

      outp = OrderedDict([[self.map['number_images'], len(dets)],
                          [self.map['number_similar_images'], nb_images_similar],
                          [self.map['all_similar_images'], output]])

    print "[format_output: log] build_output took: {}".format(time.time() - start_build_output)
    return outp
<|endoftext|>


<commit_before>class DictOutput():

  def __init__(self, mode='CamelCase'):
    self.map = dict()
    self.coord_map = ["left", "top", "right", "bottom"]
    # add your input type if you create one.
    self.input_types = ["image", "face"]

    if mode == 'CamelCase':
        self.fillDictCamelCase()
    else:
        self.fillDictOld()

  def fillInputFieldsCamelCase(self, input, input_str):
    self.map['query_' + input] = "Query" + input_str
    self.map['similar_' + input + 's'] = "Similar" + input_str + "s"
    self.map[input + 's'] = input_str + "s"
    self.map['number_'+input+'s'] = "Number" + input_str + "s"
    self.map['number_similar_'+input+'s'] = "NumberSimilar" + input_str + "s"
    self.map['all_similar_'+input+'s'] = "AllSimilar" + input_str + "s"

  def fillInputFieldsOld(self, input, input_str):
    self.map['query_' + input] = "query_" + input_str
    self.map['similar_' + input + 's'] = "similar_" + input_str + "s"
    self.map[input + 's'] = input_str + "s"
    self.map['number_'+input+'s'] = "number_" + input_str + "s"
    self.map['number_similar_'+input+'s'] = "number_similar_" + input_str + "s"
    self.map['all_similar_'+input+'s'] = "all_similar_" + input_str + "s"

  def fillDictCamelCase(self):
    self.map['query_sha1'] = "QuerySha1"
    self.map['query_url'] = "QueryURL"
    self.map['image_sha1s'] = "ImageSha1s"
    self.map['img_info'] = "ImgInfo"
    self.map['distances'] = "Distances"
    self.map['cached_image_urls'] = "CachedImageURLs"
    for input in self.input_types:
      input_str = input.title()
      self.fillInputFieldsCamelCase(input, input_str)

    # self.map['query_face'] = "QueryFace"
    # self.map['query_image'] = "QueryImage"
    # self.map['similar_faces'] = "SimilarFaces"
    # self.map['similar_images'] = "SimilarImages"
    # self.map['faces'] = "Faces"
    # self.map['images'] = "Images"
    # self.map['number_faces'] = "NumberFaces"
    # self.map['number_images'] = "NumberImages"
    # self.map['number_similar_faces'] = "NumberSimilarFaces"
    # self.map['number_similar_images'] = "NumberSimilarImages"
    # self.map['all_similar_faces'] = "AllSimilarFaces"
    # self.map['all_similar_images'] = "AllSimilarImages"

  def fillDictOld(self):
    self.map['query_sha1'] = "query_sha1"
    self.map['query_url'] = "query_url"
    self.map['image_sha1s'] = "image_sha1s"
    self.map['img_info'] = "img_info"
    self.map['distances'] = "distances"
    self.map['cached_image_urls'] = "cached_image_urls"
    for input in self.input_types:
      self.fillInputFieldsOld(input, input)

    # self.map['query_face'] = "query_face"
    # self.map['query_image'] = "query_image"
    # self.map['similar_faces'] = "similar_faces"
    # self.map['similar_images'] = "similar_images"
    # self.map['faces'] = "faces"
    # self.map['number_images'] = "number_images"
    # self.map['number_faces'] = "number_faces"
    # self.map['number_similar_faces'] = "number_similar_faces"
    # self.map['number_similar_images'] = "number_similar_images"
    # self.map['all_similar_faces'] = "all_similar_faces"
    # self.map['all_similar_images'] = "all_similar_images"

  def format_output(self, dets, sim_images, sim_dets, sim_score, options_dict=dict(), input_type="image"):
    import time
    from collections import OrderedDict
    print "[format_output: log] options are: {}".format(options_dict)
    start_build_output = time.time()
    output = []
    images_query = set()
    nb_faces_query = 0
    nb_faces_similar = 0
    nb_images_similar = 0

    #print 'dets',len(dets),dets
    #print 'sim_dets', len(sim_dets), sim_dets

    if input_type != "image":

      for i in range(len(dets)):

        # No detection in query
        if not dets[i][1]:
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:5]
          images_query.add(dets[i][0])
          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], 0],
                                                                 [self.map['image_sha1s'], []],
                                                                 [self.map[input_type+'s'], []],
                                                                 [self.map['cached_image_urls'], []],
                                                                 [self.map['distances'], []]])
          continue

        # We found some faces...
        #print len(dets[i][1])
        for j, face_bbox in enumerate(dets[i][1]):
          nb_faces_query += 1

          # Add one output for each face query
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          output[out_i][self.map['query_'+input_type]] = dets[i][1][j]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:]
          images_query.add(dets[i][0])

          nb_dets = 0
          #print "sim_dets[i]",sim_dets[i]

          if sim_dets[i]:
            if len(sim_dets[i])>j and sim_dets[i][j]:
              #print "sim_dets[i][j]",sim_dets[i][j]
              nb_dets = len(sim_dets[i][j])

          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], nb_dets],
                                                                [self.map['image_sha1s'], []],
                                                                [self.map[input_type+'s'], []],
                                                                [self.map['img_info'], []],
                                                                [self.map['cached_image_urls'], []],
                                                                [self.map['distances'], []]])

          #print 'nb_faces: %d' % nb_faces

          # Explore list of similar faces
          for jj in range(nb_dets):
            sim_det = sim_dets[i][j][jj]
            nb_faces_similar += 1
            #print sim_images[i][j][jj]
            output[out_i][self.map['similar_'+input_type+'s']][self.map['image_sha1s']].append(sim_images[i][j][jj][0].strip())
            if sim_images[i][j][jj] > 1:
              output[out_i][self.map['similar_'+input_type+'s']][self.map['cached_image_urls']].append(sim_images[i][j][jj][1][self.url_field].strip())

            tmp_face_dict = dict()
            for tfi, tfcoord in enumerate(sim_det.split('_')[1:]):
              tmp_face_dict[self.coord_map[tfi]] = int(tfcoord)
            output[out_i][self.map['similar_'+input_type+'s']][self.map[input_type+'s']].append(tmp_face_dict)
            # this is not in HBase for all/most images...
            #osf_imginfo = sim_images[i][j][jj][1][self.img_info_field].strip()
            output[out_i][self.map['similar_'+input_type+'s']][self.map['img_info']].append('')
            output[out_i][self.map['similar_'+input_type+'s']][self.map['distances']].append(sim_score[i][j][jj])

      outp = OrderedDict([[self.map['number_images'], len(images_query)],
                          [self.map['number_'+input_type+'s'], nb_faces_query], # this would overwrite number_images if input_type is image...
                          [self.map['number_similar_'+input_type+'s'], nb_faces_similar],
                          [self.map['all_similar_'+input_type+'s'], output]])

    else:
      # no detections, dets contains list of query images sha1

      for i in range(len(dets)):
        sha1 = dets[i][0]

        # Add one output for each image query
        output.append(dict())
        out_i = len(output) - 1
        output[out_i][self.map['query_sha1']] = sha1
        if dets[i][1]:
          output[out_i][self.map['query_url']] = dets[i][1]

        if sim_images[i][0]:
          nb_images = len(sim_images[i][0])

        output[out_i][self.map['similar_images']] = OrderedDict([[self.map['number_images'], nb_images],
                                                              [self.map['image_sha1s'], []],
                                                              [self.map['cached_image_urls'], []],
                                                              [self.map['distances'], []]])

        # Explore list of similar images
        for j in range(nb_images):
          #print "sim_images[i][j]",sim_images[i][0][j]
          nb_images_similar += 1
          output[out_i][self.map['similar_images']][self.map['image_sha1s']].append(sim_images[i][0][j][0].strip())
          output[out_i][self.map['similar_images']][self.map['cached_image_urls']].append(sim_images[i][0][j][1][self.url_field].strip())
          output[out_i][self.map['similar_images']][self.map['distances']].append(float(sim_score[i][0][j]))

      outp = OrderedDict([[self.map['number_images'], len(dets)],
                          [self.map['number_similar_images'], nb_images_similar],
                          [self.map['all_similar_images'], output]])

    print "[format_output: log] build_output took: {}".format(time.time() - start_build_output)
    #print outp
    return outp
<commit_msg>try to circumvent failing HBase<commit_after>class DictOutput():

  def __init__(self, mode='CamelCase'):
    self.map = dict()
    self.coord_map = ["left", "top", "right", "bottom"]
    # add your input type if you create one.
    self.input_types = ["image", "face"]

    if mode == 'CamelCase':
        self.fillDictCamelCase()
    else:
        self.fillDictOld()

  def fillInputFieldsCamelCase(self, input, input_str):
    self.map['query_' + input] = "Query" + input_str
    self.map['similar_' + input + 's'] = "Similar" + input_str + "s"
    self.map[input + 's'] = input_str + "s"
    self.map['number_'+input+'s'] = "Number" + input_str + "s"
    self.map['number_similar_'+input+'s'] = "NumberSimilar" + input_str + "s"
    self.map['all_similar_'+input+'s'] = "AllSimilar" + input_str + "s"

  def fillInputFieldsOld(self, input, input_str):
    self.map['query_' + input] = "query_" + input_str
    self.map['similar_' + input + 's'] = "similar_" + input_str + "s"
    self.map[input + 's'] = input_str + "s"
    self.map['number_'+input+'s'] = "number_" + input_str + "s"
    self.map['number_similar_'+input+'s'] = "number_similar_" + input_str + "s"
    self.map['all_similar_'+input+'s'] = "all_similar_" + input_str + "s"

  def fillDictCamelCase(self):
    self.map['query_sha1'] = "QuerySha1"
    self.map['query_url'] = "QueryURL"
    self.map['image_sha1s'] = "ImageSha1s"
    self.map['img_info'] = "ImgInfo"
    self.map['distances'] = "Distances"
    self.map['cached_image_urls'] = "CachedImageURLs"
    for input in self.input_types:
      input_str = input.title()
      self.fillInputFieldsCamelCase(input, input_str)

    # self.map['query_face'] = "QueryFace"
    # self.map['query_image'] = "QueryImage"
    # self.map['similar_faces'] = "SimilarFaces"
    # self.map['similar_images'] = "SimilarImages"
    # self.map['faces'] = "Faces"
    # self.map['images'] = "Images"
    # self.map['number_faces'] = "NumberFaces"
    # self.map['number_images'] = "NumberImages"
    # self.map['number_similar_faces'] = "NumberSimilarFaces"
    # self.map['number_similar_images'] = "NumberSimilarImages"
    # self.map['all_similar_faces'] = "AllSimilarFaces"
    # self.map['all_similar_images'] = "AllSimilarImages"

  def fillDictOld(self):
    self.map['query_sha1'] = "query_sha1"
    self.map['query_url'] = "query_url"
    self.map['image_sha1s'] = "image_sha1s"
    self.map['img_info'] = "img_info"
    self.map['distances'] = "distances"
    self.map['cached_image_urls'] = "cached_image_urls"
    for input in self.input_types:
      self.fillInputFieldsOld(input, input)

    # self.map['query_face'] = "query_face"
    # self.map['query_image'] = "query_image"
    # self.map['similar_faces'] = "similar_faces"
    # self.map['similar_images'] = "similar_images"
    # self.map['faces'] = "faces"
    # self.map['number_images'] = "number_images"
    # self.map['number_faces'] = "number_faces"
    # self.map['number_similar_faces'] = "number_similar_faces"
    # self.map['number_similar_images'] = "number_similar_images"
    # self.map['all_similar_faces'] = "all_similar_faces"
    # self.map['all_similar_images'] = "all_similar_images"

  def format_output(self, dets, sim_images, sim_dets, sim_score, options_dict=dict(), input_type="image"):
    import time
    from collections import OrderedDict
    print "[format_output: log] options are: {}".format(options_dict)
    start_build_output = time.time()
    output = []
    images_query = set()
    nb_faces_query = 0
    nb_faces_similar = 0
    nb_images_similar = 0

    #print 'dets',len(dets),dets
    #print 'sim_dets', len(sim_dets), sim_dets

    if input_type != "image":

      for i in range(len(dets)):

        # No detection in query
        if not dets[i][1]:
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:5]
          images_query.add(dets[i][0])
          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], 0],
                                                                 [self.map['image_sha1s'], []],
                                                                 [self.map[input_type+'s'], []],
                                                                 [self.map['cached_image_urls'], []],
                                                                 [self.map['distances'], []]])
          continue

        # We found some faces...
        #print len(dets[i][1])
        for j, face_bbox in enumerate(dets[i][1]):
          nb_faces_query += 1

          # Add one output for each face query
          output.append(dict())
          out_i = len(output) - 1
          output[out_i][self.map['query_sha1']] = dets[i][0]
          output[out_i][self.map['query_'+input_type]] = dets[i][1][j]
          if dets[i][2]:
            output[out_i][self.map['query_url']] = dets[i][2]
          output[out_i][self.map['img_info']] = dets[i][3:]
          images_query.add(dets[i][0])

          nb_dets = 0
          #print "sim_dets[i]",sim_dets[i]

          if sim_dets[i]:
            if len(sim_dets[i])>j and sim_dets[i][j]:
              #print "sim_dets[i][j]",sim_dets[i][j]
              nb_dets = len(sim_dets[i][j])

          output[out_i][self.map['similar_'+input_type+'s']] = OrderedDict([[self.map['number_'+input_type+'s'], nb_dets],
                                                                [self.map['image_sha1s'], []],
                                                                [self.map[input_type+'s'], []],
                                                                [self.map['img_info'], []],
                                                                [self.map['cached_image_urls'], []],
                                                                [self.map['distances'], []]])

          #print 'nb_faces: %d' % nb_faces

          # Explore list of similar faces
          for jj in range(nb_dets):
            sim_det = sim_dets[i][j][jj]
            nb_faces_similar += 1
            #print sim_images[i][j][jj]
            output[out_i][self.map['similar_'+input_type+'s']][self.map['image_sha1s']].append(sim_images[i][j][jj][0].strip())
            if sim_images[i][j][jj] > 1:
              output[out_i][self.map['similar_'+input_type+'s']][self.map['cached_image_urls']].append(sim_images[i][j][jj][1][self.url_field].strip())

            tmp_face_dict = dict()
            for tfi, tfcoord in enumerate(sim_det.split('_')[1:]):
              tmp_face_dict[self.coord_map[tfi]] = int(tfcoord)
            output[out_i][self.map['similar_'+input_type+'s']][self.map[input_type+'s']].append(tmp_face_dict)
            # this is not in HBase for all/most images...
            #osf_imginfo = sim_images[i][j][jj][1][self.img_info_field].strip()
            output[out_i][self.map['similar_'+input_type+'s']][self.map['img_info']].append('')
            output[out_i][self.map['similar_'+input_type+'s']][self.map['distances']].append(sim_score[i][j][jj])

      outp = OrderedDict([[self.map['number_images'], len(images_query)],
                          [self.map['number_'+input_type+'s'], nb_faces_query], # this would overwrite number_images if input_type is image...
                          [self.map['number_similar_'+input_type+'s'], nb_faces_similar],
                          [self.map['all_similar_'+input_type+'s'], output]])

    else:
      # no detections, dets contains list of query images sha1

      for i in range(len(dets)):
        sha1 = dets[i][0]

        # Add one output for each image query
        output.append(dict())
        out_i = len(output) - 1
        output[out_i][self.map['query_sha1']] = sha1
        if dets[i][1]:
          output[out_i][self.map['query_url']] = dets[i][1]

        if sim_images[i][0]:
          nb_images = len(sim_images[i][0])

        output[out_i][self.map['similar_images']] = OrderedDict([[self.map['number_images'], nb_images],
                                                              [self.map['image_sha1s'], []],
                                                              [self.map['cached_image_urls'], []],
                                                              [self.map['distances'], []]])

        # Explore list of similar images
        for j in range(nb_images):
          #print "sim_images[i][j]",sim_images[i][0][j]
          nb_images_similar += 1
          output[out_i][self.map['similar_images']][self.map['image_sha1s']].append(sim_images[i][0][j][0].strip())
          try:
            output[out_i][self.map['similar_images']][self.map['cached_image_urls']].append(sim_images[i][0][j][1][self.url_field].strip())
          except Exception:
            output[out_i][self.map['similar_images']][self.map['cached_image_urls']].append("")
          output[out_i][self.map['similar_images']][self.map['distances']].append(float(sim_score[i][0][j]))

      outp = OrderedDict([[self.map['number_images'], len(dets)],
                          [self.map['number_similar_images'], nb_images_similar],
                          [self.map['all_similar_images'], output]])

    print "[format_output: log] build_output took: {}".format(time.time() - start_build_output)
    #print outp
    return outp
<|endoftext|>


<commit_before>import unittest
import os

import factory

from cumulusci.utils import temporary_dir
from cumulusci.tasks.bulkdata.tests.test_bulkdata import _make_task
from cumulusci.tasks.bulkdata.tests.dummy_data_factory import (
    GenerateDummyData,
    ContactFactory,
)
from cumulusci.tasks.bulkdata import factory_utils

class TestFactoryUtils(unittest.TestCase):
    def test_factory(self):
        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")

        with temporary_dir() as d:
            tmp_db_path = os.path.join(d, "temp.db")
            dburl = "sqlite:///" + tmp_db_path
            task = _make_task(
                GenerateDummyData,
                {
                    "options": {
                        "num_records": 10,
                        "mapping": mapping_file,
                        "database_url": dburl,
                    }
                },
            )
            task()

class TestAdder(unittest.TestCase):
    def test_adder(self):
        a = factory_utils.Adder(10)
        b = a(20)
        assert b == 30
        c = a(0)
        assert c == 30
        d = a(-5)
        assert d == 25
        a.reset(3)
        assert a(0) == 3

class TestFactories(unittest.TestCase):
    def test_factories(self):
        class Broken(factory.alchemy.SQLAlchemyModelFactory):
            class Meta:
                model = "xyzzy"

        with self.assertRaises(KeyError):
            factory_utils.Factories(None, {}, {"A": ContactFactory, "B": Broken})
<commit_msg>Generate num_records divisible by 4<commit_after>import unittest
import os

import factory

from cumulusci.utils import temporary_dir
from cumulusci.tasks.bulkdata.tests.test_bulkdata import _make_task
from cumulusci.tasks.bulkdata.tests.dummy_data_factory import (
    GenerateDummyData,
    ContactFactory,
)
from cumulusci.tasks.bulkdata import factory_utils

class TestFactoryUtils(unittest.TestCase):
    def test_factory(self):
        mapping_file = os.path.join(os.path.dirname(__file__), "mapping_vanilla_sf.yml")

        with temporary_dir() as d:
            tmp_db_path = os.path.join(d, "temp.db")
            dburl = "sqlite:///" + tmp_db_path
            task = _make_task(
                GenerateDummyData,
                {
                    "options": {
                        "num_records": 12,
                        "mapping": mapping_file,
                        "database_url": dburl,
                    }
                },
            )
            task()

class TestAdder(unittest.TestCase):
    def test_adder(self):
        a = factory_utils.Adder(10)
        b = a(20)
        assert b == 30
        c = a(0)
        assert c == 30
        d = a(-5)
        assert d == 25
        a.reset(3)
        assert a(0) == 3

class TestFactories(unittest.TestCase):
    def test_factories(self):
        class Broken(factory.alchemy.SQLAlchemyModelFactory):
            class Meta:
                model = "xyzzy"

        with self.assertRaises(KeyError):
            factory_utils.Factories(None, {}, {"A": ContactFactory, "B": Broken})
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Captures application state on a breakpoint hit."""

import copy
import datetime
import inspect
import types

import cdbg_native as native

_VECTOR_TYPES = {types.TupleType, types.ListType, types.SliceType, set}

EMPTY_DICTIONARY = 'Empty dictionary'
EMPTY_COLLECTION = 'Empty collection'
OBJECT_HAS_NO_FIELDS = 'Object has no fields'

class CaptureCollector(object):
  """Captures application state snapshot.

  Captures call stack, local variables and referenced objects. Then formats the
  result to be sent back to the user.

  The performance of this class is important. Once the breakpoint hits, the
  completion of the user request will be delayed until the collection is over.
  It might make sense to implement this logic in C++.

  Attributes:
    breakpoint: breakpoint definition augmented with captured call stack,
        local variables, arguments and referenced objects.
  """

  # Additional type-specific printers. Each pretty printer is a callable
  # that returns None if it doesn't recognize the object or returns iterable
  # enumerating object fields (name-value tuple).
  pretty_printers = []

  def __init__(self, definition):
    """Class constructor.

    Args:
      definition: breakpoint definition that this class will augment with
          captured data.
    """
    self.breakpoint = copy.deepcopy(definition)

    self.breakpoint['stackFrames'] = []
    self.breakpoint['evaluatedExpressions'] = []
    self.breakpoint['variableTable'] = [{
        'status': {
            'isError': True,
            'refersTo': 'VARIABLE_VALUE',
            'description': {'format': 'Buffer full'}}}]

    # Shortcut to variables table in the breakpoint message.
    self._var_table = self.breakpoint['variableTable']

    # Maps object ID to its index in variables table.
    self._var_table_index = {}

    # Total size of data collected so far. Limited by max_size.
    self._total_size = 0

    # Maximum number of stack frame to capture. The limit is aimed to reduce
    # the overall collection time.
    self.max_frames = 20

    # Only collect locals and arguments on the few top frames. For the rest of
    # the frames we only collect the source location.
    self.max_expand_frames = 5

    # Maximum amount of data to capture. The application will usually have a
    # lot of objects and we need to stop somewhere to keep the delay
    # reasonable.
    # This constant only counts the collected payload. Overhead due to key
    # names is not counted.
    self.max_size = 32768  # 32 KB

    # Maximum number of character to allow for a single value. Longer strings
    # are truncated.
    self.max_value_len = 256

    # Maximum number of items in a list to capture.
    self.max_list_items = 25

    # Maximum depth of dictionaries to capture.
    self.max_depth = 5

  def Collect(self, top_frame):
    """Collects call stack, local variables and objects.

    Starts collection from the specified frame. We don't start from the top
    frame to exclude the frames due to debugger. Updates the content of
    self.breakpoint.

    Args:
      top_frame: top frame to start data collection.
    """
    # Evaluate call stack.
    frame = top_frame
    breakpoint_frames = self.breakpoint['stackFrames']
    while frame and (len(breakpoint_frames) < self.max_frames):
      code = frame.f_code
      if len(breakpoint_frames) < self.max_expand_frames:
        frame_arguments, frame_locals = self.CaptureFrameLocals(frame)
      else:
        frame_arguments = []
        frame_locals = []

      breakpoint_frames.append({
          'function': code.co_name,
          'location': {
              'path': code.co_filename,
              'line': frame.f_lineno},
          'arguments': frame_arguments,
          'locals': frame_locals})
      frame = frame.f_back

    # Evaluate watched expressions.
    if 'expressions' in self.breakpoint:
      self.breakpoint['evaluatedExpressions'] = [
          self._EvaluateExpression(top_frame, expression) for expression
          in self.breakpoint['expressions']]

    # Explore variables table in BFS fashion. The variables table will grow
    # inside CaptureVariable as we encounter new references.
    i = 1
    while (i < len(self._var_table)) and (self._total_size < self.max_size):
      self._var_table[i] = self.CaptureVariable(self._var_table[i], 0, False)
      i += 1

    # Trim variables table and change make all references to variables that
    # didn't make it point to var_index of 0 ("buffer full")
    self.TrimVariableTable(i)

  def CaptureFrameLocals(self, frame):
    """Captures local variables and arguments of the specified frame.

    Args:
      frame: frame to capture locals and arguments.

    Returns:
      (arguments, locals) tuple.
    """
    # Capture all local variables (including method arguments).
    variables = {n: self.CaptureNamedVariable(n, v)
                 for n, v in frame.f_locals.viewitems()}

    # Split between locals and arguments (keeping arguments in the right order).
    nargs = frame.f_code.co_argcount
    if frame.f_code.co_flags & inspect.CO_VARARGS: nargs += 1
    if frame.f_code.co_flags & inspect.CO_VARKEYWORDS: nargs += 1

    frame_arguments = []
    for argname in frame.f_code.co_varnames[:nargs]:
      if argname in variables: frame_arguments.append(variables.pop(argname))

    return (frame_arguments, list(variables.viewvalues()))

  def CaptureNamedVariable(self, name, value, depth=1):
    """Appends name to the product of CaptureVariable.

    Args:
      name: name of the variable.
      value: data to capture
      depth: nested depth of dictionaries and vectors so far.

    Returns:
      Formatted captured data as per Variable proto with name.
    """
    if not hasattr(name, '__dict__'):
      name = str(name)
    else:  # TODO(vlif): call str(name) with immutability verifier here.
      name = str(id(name))
    self._total_size += len(name)

    v = self.CaptureVariable(value, depth)
    v['name'] = name
    return v

  def CaptureVariablesList(self, items, depth, empty_message):
    """Captures list of named items.

    Args:
      items: iterable of (name, value) tuples.
      depth: nested depth of dictionaries and vectors for items.
      empty_message: info status message to set if items is empty.

    Returns:
      List of formatted variable objects.
    """
    v = []
    for name, value in items:
      if (self._total_size >= self.max_size) or (len(v) >= self.max_list_items):
        v.append({
            'status': {
                'refers_to': 'VARIABLE_VALUE',
                'description': {
                    'format': 'Only first $0 items were captured',
                    'parameters': [str(len(v))]}}})
        break
      v.append(self.CaptureNamedVariable(name, value, depth))

    if not v:
      return [{'status': {
          'is_error': False,
          'refers_to': 'VARIABLE_NAME',
          'description': {'format': empty_message}}}]

    return v

  def CaptureVariable(self, value, depth=1, can_enqueue=True):
    """Captures a single nameless object into Variable message.

    TODO(vlif): safely evaluate iterable types.
    TODO(vlif): safely call str(value)

    Args:
      value: data to capture
      depth: nested depth of dictionaries and vectors so far.
      can_enqueue: allows referencing the object in variables table.

    Returns:
      Formatted captured data as per Variable proto.
    """
    if depth == self.max_depth:
      return {'varTableIndex': 0}  # Buffer full.

    if value is None:
      self._total_size += 4
      return {'value': 'None'}

    if isinstance(value, (int, long, float, complex, str, unicode, bool)):
      r = self.TrimString(repr(value))  # Primitive type, always immutable.
      self._total_size += len(r)
      return {'value': r}

    if isinstance(value, (datetime.date, datetime.time, datetime.timedelta)):
      r = str(value)  # Safe to call str().
      self._total_size += len(r)
      return {'value': r}

    if type(value) is dict:
      return {'members': self.CaptureVariablesList(value.iteritems(),
                                                   depth + 1,
                                                   EMPTY_DICTIONARY)}

    if type(value) in _VECTOR_TYPES:
      return {'members': self.CaptureVariablesList(
          (('[%d]' % i, x) for i, x in enumerate(value)),
          depth + 1,
          EMPTY_COLLECTION)}

    if type(value) is types.FunctionType:
      self._total_size += len(value.func_name)
      return {'value': 'function ' + value.func_name}

    if can_enqueue:
      index = self._var_table_index.get(id(value))
      if index is None:
        index = len(self._var_table)
        self._var_table_index[id(value)] = index
        self._var_table.append(value)
      self._total_size += 4  # number of characters to accomodate a number.
      return {'varTableIndex': index}

    for pretty_printer in CaptureCollector.pretty_printers:
      fields = pretty_printer(value)
      if fields:
        return {'members': self.CaptureVariablesList(fields,
                                                     depth + 1,
                                                     OBJECT_HAS_NO_FIELDS)}

    if not hasattr(value, '__dict__'):
      r = str(type(value))
      self._total_size += len(r)
      return {'value': r}

    if not value.__dict__:
      return {'members': [{'status': {
          'is_error': False,
          'refers_to': 'VARIABLE_NAME',
          'description': {'format': OBJECT_HAS_NO_FIELDS}}}]}

    return self.CaptureVariable(value.__dict__, depth + 1)

  def _EvaluateExpression(self, frame, expression):
    """Compiles and evaluates watched expression to a Variable message.

    Args:
      frame: evaluation context.
      expression: watched expression to compile and evaluate.

    Returns:
      Formatted object corresponding to the evaluation result.
    """
    try:
      code = compile(expression, '<watched_expression>', 'eval')
    except TypeError as e:  # condition string contains null bytes.
      return {
          'name': expression,
          'status': {
              'isError': True,
              'refersTo': 'VARIABLE_NAME',
              'description': {
                  'format': 'Invalid expression',
                  'parameters': [str(e)]}}}
    except SyntaxError as e:
      return {
          'name': expression,
          'status': {
              'isError': True,
              'refersTo': 'VARIABLE_NAME',
              'description': {
                  'format': 'Expression could not be compiled: $0',
                  'parameters': [e.msg]}}}

    try:
      value = native.CallImmutable(frame, code)
    except BaseException as e:
      return {
          'name': expression,
          'status': {
              'isError': True,
              'refersTo': 'VARIABLE_VALUE',
              'description': {
                  'format': 'Exception occurred: $0',
                  'parameters': [e.message]}}}
    return self.CaptureNamedVariable(expression, value)

  def TrimString(self, s):
    """Trims the string if it exceeds max_value_len."""
    if len(s) <= self.max_value_len:
      return s
    return s[:self.max_value_len] + '...'

  def TrimVariableTable(self, new_size):
    """Trims the variable table in the formatted breakpoint message.

    Removes trailing entries in variables table. Then scans the entire
    breakpoint message and replaces references to the trimmed variables to
    point to var_index of 0 ("buffer full").

    Args:
      new_size: desired size of variables table.
    """

    def ProcessBufferFull(variables):
      for variable in variables:
        var_index = variable.get('varTableIndex')
        if var_index is not None and (var_index >= new_size):
          variable['varTableIndex'] = 0  # Buffer full.
        members = variable.get('members')
        if members is not None:
          ProcessBufferFull(members)

    del self._var_table[new_size:]
    for stack_frame in self.breakpoint['stackFrames']:
      ProcessBufferFull(stack_frame['arguments'])
      ProcessBufferFull(stack_frame['locals'])
    ProcessBufferFull(self._var_table)
    ProcessBufferFull(self.breakpoint['evaluatedExpressions'])
<commit_msg>Return relative paths to the source location instead of absolute paths. This fixes stack frames navigation with Code Search. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=105798447<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Captures application state on a breakpoint hit."""

import copy
import datetime
import inspect
import os
import sys
import types

import cdbg_native as native

_VECTOR_TYPES = {types.TupleType, types.ListType, types.SliceType, set}

EMPTY_DICTIONARY = 'Empty dictionary'
EMPTY_COLLECTION = 'Empty collection'
OBJECT_HAS_NO_FIELDS = 'Object has no fields'

class CaptureCollector(object):
  """Captures application state snapshot.

  Captures call stack, local variables and referenced objects. Then formats the
  result to be sent back to the user.

  The performance of this class is important. Once the breakpoint hits, the
  completion of the user request will be delayed until the collection is over.
  It might make sense to implement this logic in C++.

  Attributes:
    breakpoint: breakpoint definition augmented with captured call stack,
        local variables, arguments and referenced objects.
  """

  # Additional type-specific printers. Each pretty printer is a callable
  # that returns None if it doesn't recognize the object or returns iterable
  # enumerating object fields (name-value tuple).
  pretty_printers = []

  def __init__(self, definition):
    """Class constructor.

    Args:
      definition: breakpoint definition that this class will augment with
          captured data.
    """
    self.breakpoint = copy.deepcopy(definition)

    self.breakpoint['stackFrames'] = []
    self.breakpoint['evaluatedExpressions'] = []
    self.breakpoint['variableTable'] = [{
        'status': {
            'isError': True,
            'refersTo': 'VARIABLE_VALUE',
            'description': {'format': 'Buffer full'}}}]

    # Shortcut to variables table in the breakpoint message.
    self._var_table = self.breakpoint['variableTable']

    # Maps object ID to its index in variables table.
    self._var_table_index = {}

    # Total size of data collected so far. Limited by max_size.
    self._total_size = 0

    # Maximum number of stack frame to capture. The limit is aimed to reduce
    # the overall collection time.
    self.max_frames = 20

    # Only collect locals and arguments on the few top frames. For the rest of
    # the frames we only collect the source location.
    self.max_expand_frames = 5

    # Maximum amount of data to capture. The application will usually have a
    # lot of objects and we need to stop somewhere to keep the delay
    # reasonable.
    # This constant only counts the collected payload. Overhead due to key
    # names is not counted.
    self.max_size = 32768  # 32 KB

    # Maximum number of character to allow for a single value. Longer strings
    # are truncated.
    self.max_value_len = 256

    # Maximum number of items in a list to capture.
    self.max_list_items = 25

    # Maximum depth of dictionaries to capture.
    self.max_depth = 5

  def Collect(self, top_frame):
    """Collects call stack, local variables and objects.

    Starts collection from the specified frame. We don't start from the top
    frame to exclude the frames due to debugger. Updates the content of
    self.breakpoint.

    Args:
      top_frame: top frame to start data collection.
    """
    # Evaluate call stack.
    frame = top_frame
    breakpoint_frames = self.breakpoint['stackFrames']
    while frame and (len(breakpoint_frames) < self.max_frames):
      code = frame.f_code
      if len(breakpoint_frames) < self.max_expand_frames:
        frame_arguments, frame_locals = self.CaptureFrameLocals(frame)
      else:
        frame_arguments = []
        frame_locals = []

      breakpoint_frames.append({
          'function': code.co_name,
          'location': {
              'path': CaptureCollector._NormalizePath(code.co_filename),
              'line': frame.f_lineno},
          'arguments': frame_arguments,
          'locals': frame_locals})
      frame = frame.f_back

    # Evaluate watched expressions.
    if 'expressions' in self.breakpoint:
      self.breakpoint['evaluatedExpressions'] = [
          self._EvaluateExpression(top_frame, expression) for expression
          in self.breakpoint['expressions']]

    # Explore variables table in BFS fashion. The variables table will grow
    # inside CaptureVariable as we encounter new references.
    i = 1
    while (i < len(self._var_table)) and (self._total_size < self.max_size):
      self._var_table[i] = self.CaptureVariable(self._var_table[i], 0, False)
      i += 1

    # Trim variables table and change make all references to variables that
    # didn't make it point to var_index of 0 ("buffer full")
    self.TrimVariableTable(i)

  def CaptureFrameLocals(self, frame):
    """Captures local variables and arguments of the specified frame.

    Args:
      frame: frame to capture locals and arguments.

    Returns:
      (arguments, locals) tuple.
    """
    # Capture all local variables (including method arguments).
    variables = {n: self.CaptureNamedVariable(n, v)
                 for n, v in frame.f_locals.viewitems()}

    # Split between locals and arguments (keeping arguments in the right order).
    nargs = frame.f_code.co_argcount
    if frame.f_code.co_flags & inspect.CO_VARARGS: nargs += 1
    if frame.f_code.co_flags & inspect.CO_VARKEYWORDS: nargs += 1

    frame_arguments = []
    for argname in frame.f_code.co_varnames[:nargs]:
      if argname in variables: frame_arguments.append(variables.pop(argname))

    return (frame_arguments, list(variables.viewvalues()))

  def CaptureNamedVariable(self, name, value, depth=1):
    """Appends name to the product of CaptureVariable.

    Args:
      name: name of the variable.
      value: data to capture
      depth: nested depth of dictionaries and vectors so far.

    Returns:
      Formatted captured data as per Variable proto with name.
    """
    if not hasattr(name, '__dict__'):
      name = str(name)
    else:  # TODO(vlif): call str(name) with immutability verifier here.
      name = str(id(name))
    self._total_size += len(name)

    v = self.CaptureVariable(value, depth)
    v['name'] = name
    return v

  def CaptureVariablesList(self, items, depth, empty_message):
    """Captures list of named items.

    Args:
      items: iterable of (name, value) tuples.
      depth: nested depth of dictionaries and vectors for items.
      empty_message: info status message to set if items is empty.

    Returns:
      List of formatted variable objects.
    """
    v = []
    for name, value in items:
      if (self._total_size >= self.max_size) or (len(v) >= self.max_list_items):
        v.append({
            'status': {
                'refers_to': 'VARIABLE_VALUE',
                'description': {
                    'format': 'Only first $0 items were captured',
                    'parameters': [str(len(v))]}}})
        break
      v.append(self.CaptureNamedVariable(name, value, depth))

    if not v:
      return [{'status': {
          'is_error': False,
          'refers_to': 'VARIABLE_NAME',
          'description': {'format': empty_message}}}]

    return v

  def CaptureVariable(self, value, depth=1, can_enqueue=True):
    """Captures a single nameless object into Variable message.

    TODO(vlif): safely evaluate iterable types.
    TODO(vlif): safely call str(value)

    Args:
      value: data to capture
      depth: nested depth of dictionaries and vectors so far.
      can_enqueue: allows referencing the object in variables table.

    Returns:
      Formatted captured data as per Variable proto.
    """
    if depth == self.max_depth:
      return {'varTableIndex': 0}  # Buffer full.

    if value is None:
      self._total_size += 4
      return {'value': 'None'}

    if isinstance(value, (int, long, float, complex, str, unicode, bool)):
      r = self.TrimString(repr(value))  # Primitive type, always immutable.
      self._total_size += len(r)
      return {'value': r}

    if isinstance(value, (datetime.date, datetime.time, datetime.timedelta)):
      r = str(value)  # Safe to call str().
      self._total_size += len(r)
      return {'value': r}

    if type(value) is dict:
      return {'members': self.CaptureVariablesList(value.iteritems(),
                                                   depth + 1,
                                                   EMPTY_DICTIONARY)}

    if type(value) in _VECTOR_TYPES:
      return {'members': self.CaptureVariablesList(
          (('[%d]' % i, x) for i, x in enumerate(value)),
          depth + 1,
          EMPTY_COLLECTION)}

    if type(value) is types.FunctionType:
      self._total_size += len(value.func_name)
      return {'value': 'function ' + value.func_name}

    if can_enqueue:
      index = self._var_table_index.get(id(value))
      if index is None:
        index = len(self._var_table)
        self._var_table_index[id(value)] = index
        self._var_table.append(value)
      self._total_size += 4  # number of characters to accomodate a number.
      return {'varTableIndex': index}

    for pretty_printer in CaptureCollector.pretty_printers:
      fields = pretty_printer(value)
      if fields:
        return {'members': self.CaptureVariablesList(fields,
                                                     depth + 1,
                                                     OBJECT_HAS_NO_FIELDS)}

    if not hasattr(value, '__dict__'):
      r = str(type(value))
      self._total_size += len(r)
      return {'value': r}

    if not value.__dict__:
      return {'members': [{'status': {
          'is_error': False,
          'refers_to': 'VARIABLE_NAME',
          'description': {'format': OBJECT_HAS_NO_FIELDS}}}]}

    return self.CaptureVariable(value.__dict__, depth + 1)

  def _EvaluateExpression(self, frame, expression):
    """Compiles and evaluates watched expression to a Variable message.

    Args:
      frame: evaluation context.
      expression: watched expression to compile and evaluate.

    Returns:
      Formatted object corresponding to the evaluation result.
    """
    try:
      code = compile(expression, '<watched_expression>', 'eval')
    except TypeError as e:  # condition string contains null bytes.
      return {
          'name': expression,
          'status': {
              'isError': True,
              'refersTo': 'VARIABLE_NAME',
              'description': {
                  'format': 'Invalid expression',
                  'parameters': [str(e)]}}}
    except SyntaxError as e:
      return {
          'name': expression,
          'status': {
              'isError': True,
              'refersTo': 'VARIABLE_NAME',
              'description': {
                  'format': 'Expression could not be compiled: $0',
                  'parameters': [e.msg]}}}

    try:
      value = native.CallImmutable(frame, code)
    except BaseException as e:
      return {
          'name': expression,
          'status': {
              'isError': True,
              'refersTo': 'VARIABLE_VALUE',
              'description': {
                  'format': 'Exception occurred: $0',
                  'parameters': [e.message]}}}
    return self.CaptureNamedVariable(expression, value)

  def TrimString(self, s):
    """Trims the string if it exceeds max_value_len."""
    if len(s) <= self.max_value_len:
      return s
    return s[:self.max_value_len] + '...'

  def TrimVariableTable(self, new_size):
    """Trims the variable table in the formatted breakpoint message.

    Removes trailing entries in variables table. Then scans the entire
    breakpoint message and replaces references to the trimmed variables to
    point to var_index of 0 ("buffer full").

    Args:
      new_size: desired size of variables table.
    """

    def ProcessBufferFull(variables):
      for variable in variables:
        var_index = variable.get('varTableIndex')
        if var_index is not None and (var_index >= new_size):
          variable['varTableIndex'] = 0  # Buffer full.
        members = variable.get('members')
        if members is not None:
          ProcessBufferFull(members)

    del self._var_table[new_size:]
    for stack_frame in self.breakpoint['stackFrames']:
      ProcessBufferFull(stack_frame['arguments'])
      ProcessBufferFull(stack_frame['locals'])
    ProcessBufferFull(self._var_table)
    ProcessBufferFull(self.breakpoint['evaluatedExpressions'])

  @staticmethod
  def _NormalizePath(path):
    """Converts an absolute path to a relative one.

    Python keeps almost all paths absolute. This is not what we actually
    want to return. This loops through system paths (directories in which
    Python will load modules). If "path" is relative to one of them, the
    directory prefix is removed.

    Args:
      path: absolute path to normalize (relative paths will not be altered)

    Returns:
      Relative path if "path" is within one of the sys.path directories or
      the input otherwise.
    """
    path = os.path.normpath(path)

    for sys_path in sys.path:
      if not sys_path:
        continue

      # Append '/' at the end of the path if it's not there already.
      sys_path = os.path.join(sys_path, '')

      if path.startswith(sys_path):
        return path[len(sys_path):]

    return path
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import tempfile

from corehq.apps.locations.models import SQLLocation
from corehq.apps.userreports.models import StaticDataSourceConfiguration, get_datasource_config
from corehq.apps.userreports.util import get_table_name

from custom.icds_reports.const import DASHBOARD_DOMAIN
from custom.icds_reports.models import CcsRecordMonthly, ChildHealthMonthly, AggAwc

class MBTHelper(object):

    def __init__(self, state_id, month):
        self.state_id = state_id
        self.month = month

    @property
    def domain(self):
        # Currently its only possible for one domain to have access to the ICDS dashboard per env
        return DASHBOARD_DOMAIN

    @property
    def columns(self):
        raise NotImplementedError

    @property
    def base_tablename(self):
        return self.base_class._meta.db_table

    @property
    def state_name(self):
        state = SQLLocation.objects.get(location_id=self.state_id)
        return state.name

    @property
    def output_file(self):
        temp_dir = tempfile.gettempdir()
        return '{}/{}_{}_{}.csv'.format(temp_dir, self.base_tablename, self.state_name, self.month)

    @property
    def location_columns(self):
        return ('awc.state_name', 'awc.district_name', 'awc.block_name', 'awc.awc_name', 'awc.awc_site_code')

    def query(self):
        return """
        COPY (SELECT {columns} FROM {table} t LEFT JOIN awc_location awc on t.awc_id=awc.doc_id WHERE awc.state_id='{state_id}' AND t.month='{month}') TO STDOUT WITH CSV HEADER;
        """.format(
            columns=','.join(self.columns + self.location_columns),
            table=self.base_tablename,
            state_id=self.state_id,
            month=self.month
        )

class CcsMbtHelper(MBTHelper):
    base_class = CcsRecordMonthly

    @property
    def columns(self):
        return ('t.awc_id',
                'case_id',
                'month',
                'age_in_months',
                'ccs_status',
                'open_in_month',
                'alive_in_month',
                'trimester',
                'num_rations_distributed',
                'thr_eligible',
                'tetanus_complete',
                'delivered_in_month',
                'anc1_received_at_delivery',
                'anc2_received_at_delivery',
                'anc3_received_at_delivery',
                'anc4_received_at_delivery',
                'registration_trimester_at_delivery',
                'using_ifa',
                'ifa_consumed_last_seven_days',
                'anemic_severe',
                'anemic_moderate',
                'anemic_normal',
                'anemic_unknown',
                'extra_meal',
                'resting_during_pregnancy',
                'bp_visited_in_month',
                'pnc_visited_in_month',
                'trimester_2',
                'trimester_3',
                'counsel_immediate_bf',
                'counsel_bp_vid',
                'counsel_preparation',
                'counsel_fp_vid',
                'counsel_immediate_conception',
                'counsel_accessible_postpartum_fp',
                'bp1_complete',
                'bp2_complete',
                'bp3_complete',
                'pnc_complete',
                'postnatal',
                'has_aadhar_id',
                'counsel_fp_methods',
                'pregnant',
                'pregnant_all',
                'lactating',
                'lactating_all',
                'institutional_delivery_in_month',
                'add',
                'anc_in_month',
                'caste',
                'disabled',
                'minority',
                'resident',
                'anc_weight',
                'anc_blood_pressure',
                'bp_sys',
                'bp_dia',
                'anc_hemoglobin',
                'bleeding',
                'swelling',
                'blurred_vision',
                'convulsions',
                'rupture',
                'anemia',
                'eating_extra',
                'resting',
                'immediate_breastfeeding',
                'edd',
                'delivery_nature',
                'is_ebf',
                'breastfed_at_birth',
                'anc_1',
                'anc_2',
                'anc_3',
                'anc_4',
                'tt_1',
                'tt_2',
                'valid_in_month',
                'mobile_number',
                'preg_order',
                'home_visit_date',
                'num_pnc_visits',
                'last_date_thr',
                'num_anc_complete',
                'opened_on',
                'valid_visits',
                'dob',
                'date_death')

class ChildHealthMbtHelper(MBTHelper):
    base_class = ChildHealthMonthly

    @property
    def person_case_ucr_tablename(self):
        doc_id = StaticDataSourceConfiguration.get_doc_id(self.domain, 'static-person_cases_v2')
        config, _ = get_datasource_config(doc_id, self.domain)
        return get_table_name(self.domain, config.table_id)

    @property
    def ccs_record_monthly_ucr_tablename(self):
        doc_id = StaticDataSourceConfiguration.get_doc_id(self.domain, 'static-ccs_record_cases_monthly_tableau_v2')
        config, _ = get_datasource_config(doc_id, self.domain)
        return get_table_name(self.domain, config.table_id)

    @property
    def columns(self):
        return ('t.awc_id',
                't.case_id',
                't.month',
                't.age_in_months',
                't.open_in_month',
                't.alive_in_month',
                't.wer_eligible',
                't.nutrition_status_last_recorded',
                't.current_month_nutrition_status',
                't.nutrition_status_weighed',
                't.num_rations_distributed',
                't.pse_eligible',
                't.pse_days_attended',
                't.born_in_month',
                't.low_birth_weight_born_in_month',
                't.bf_at_birth_born_in_month',
                't.ebf_eligible',
                't.ebf_in_month',
                't.ebf_not_breastfeeding_reason',
                't.ebf_drinking_liquid',
                't.ebf_eating',
                't.ebf_no_bf_no_milk',
                't.ebf_no_bf_pregnant_again',
                't.ebf_no_bf_child_too_old',
                't.ebf_no_bf_mother_sick',
                't.cf_eligible',
                't.cf_in_month',
                't.cf_diet_diversity',
                't.cf_diet_quantity',
                't.cf_handwashing',
                't.cf_demo',
                't.fully_immunized_eligible',
                't.fully_immunized_on_time',
                't.fully_immunized_late',
                't.counsel_ebf',
                't.counsel_adequate_bf',
                't.counsel_pediatric_ifa',
                't.counsel_comp_feeding_vid',
                't.counsel_increase_food_bf',
                't.counsel_manage_breast_problems',
                't.counsel_skin_to_skin',
                't.counsel_immediate_breastfeeding',
                't.recorded_weight',
                't.recorded_height',
                't.has_aadhar_id',
                't.thr_eligible',
                't.pnc_eligible',
                't.cf_initiation_in_month',
                't.cf_initiation_eligible',
                't.height_measured_in_month',
                't.current_month_stunting',
                't.stunting_last_recorded',
                't.wasting_last_recorded',
                't.current_month_wasting',
                't.valid_in_month',
                't.valid_all_registered_in_month',
                't.ebf_no_info_recorded',
                't.dob',
                't.sex',
                't.age_tranche',
                't.caste',
                't.disabled',
                't.minority',
                't.resident',
                't.immunization_in_month',
                't.days_ration_given_child',
                't.zscore_grading_hfa',
                't.zscore_grading_hfa_recorded_in_month',
                't.zscore_grading_wfh',
                't.zscore_grading_wfh_recorded_in_month',
                't.muac_grading',
                'ccs.case_id as ccs_record_case_id',
                't.date_death')

    def query(self):
        return """
        COPY (SELECT {columns} FROM {table} t
        LEFT JOIN awc_location awc on t.awc_id=awc.doc_id
        LEFT JOIN "{person_cases_ucr}" mother on mother.doc_id=t.mother_case_id
          AND awc.state_id = mother.state_id
          AND lower(substring(mother.state_id, '.{{3}}$'::text)) = '{state_id_last_3}'
        LEFT JOIN "ccs_record_monthly_{month}" ccs on ccs.person_case_id=mother.doc_id AND ccs.add=t.dob
        WHERE awc.state_id='{state_id}' AND t.month='{month}')
        TO STDOUT WITH CSV HEADER;
        """.format(
            columns=','.join(self.columns + self.location_columns),
            table=self.base_tablename,
            state_id=self.state_id,
            month=self.month,
            person_cases_ucr=self.person_case_ucr_tablename,
            ccs_record_monthly=self.ccs_record_monthly_ucr_tablename,
            state_id_last_3=self.state_id[-3:]
        )

class AwcMbtHelper(MBTHelper):
    base_class = AggAwc

    @property
    def columns(self):
        return ('t.state_id',
                't.district_id',
                't.block_id',
                't.supervisor_id',
                't.awc_id',
                'month',
                'num_awcs',
                'awc_days_open',
                'total_eligible_children',
                'total_attended_children',
                'pse_avg_attendance_percent',
                'pse_full',
                'pse_partial',
                'pse_non',
                'pse_score',
                'awc_days_provided_breakfast',
                'awc_days_provided_hotmeal',
                'awc_days_provided_thr',
                'awc_days_provided_pse',
                'awc_not_open_holiday',
                'awc_not_open_festival',
                'awc_not_open_no_help',
                'awc_not_open_department_work',
                'awc_not_open_other',
                'awc_num_open',
                'awc_not_open_no_data',
                'wer_weighed',
                'wer_eligible',
                'wer_score',
                'thr_eligible_child',
                'thr_rations_21_plus_distributed_child',
                'thr_eligible_ccs',
                'thr_rations_21_plus_distributed_ccs',
                'thr_score',
                'awc_score',
                'num_awc_rank_functional',
                'num_awc_rank_semi',
                'num_awc_rank_non',
                'cases_ccs_pregnant',
                'cases_ccs_lactating',
                'cases_child_health',
                'usage_num_pse',
                'usage_num_gmp',
                'usage_num_thr',
                'usage_num_home_visit',
                'usage_num_bp_tri1',
                'usage_num_bp_tri2',
                'usage_num_bp_tri3',
                'usage_num_pnc',
                'usage_num_ebf',
                'usage_num_cf',
                'usage_num_delivery',
                'usage_num_due_list_ccs',
                'usage_num_due_list_child_health',
                'usage_awc_num_active',
                'usage_time_pse',
                'usage_time_gmp',
                'usage_time_bp',
                'usage_time_pnc',
                'usage_time_ebf',
                'usage_time_cf',
                'usage_time_of_day_pse',
                'usage_time_of_day_home_visit',
                'vhnd_immunization',
                'vhnd_anc',
                'vhnd_gmp',
                'vhnd_num_pregnancy',
                'vhnd_num_lactating',
                'vhnd_num_mothers_6_12',
                'vhnd_num_mothers_12',
                'vhnd_num_fathers',
                'ls_supervision_visit',
                'ls_num_supervised',
                'ls_awc_location_long',
                'ls_awc_location_lat',
                'ls_awc_present',
                'ls_awc_open',
                'ls_awc_not_open_aww_not_available',
                'ls_awc_not_open_closed_early',
                'ls_awc_not_open_holiday',
                'ls_awc_not_open_unknown',
                'ls_awc_not_open_other',
                'infra_last_update_date',
                'infra_type_of_building',
                'infra_type_of_building_pucca',
                'infra_type_of_building_semi_pucca',
                'infra_type_of_building_kuccha',
                'infra_type_of_building_partial_covered_space',
                'infra_clean_water',
                'infra_functional_toilet',
                'infra_baby_weighing_scale',
                'infra_flat_weighing_scale',
                'infra_adult_weighing_scale',
                'infra_cooking_utensils',
                'infra_medicine_kits',
                'infra_adequate_space_pse',
                'cases_person_beneficiary',
                'cases_person_referred',
                'awc_days_pse_conducted',
                'num_awc_infra_last_update',
                'cases_person_has_aadhaar_v2',
                'cases_person_beneficiary_v2',
                'electricity_awc',
                'infantometer',
                'stadiometer',
                'num_anc_visits',
                'num_children_immunized',
                'usage_num_hh_reg',
                'usage_num_add_person',
                'usage_num_add_pregnancy',
                'is_launched',
                'training_phase',
                'trained_phase_1',
                'trained_phase_2',
                'trained_phase_3',
                'trained_phase_4',
                't.aggregation_level',
                'num_launched_states',
                'num_launched_districts',
                'num_launched_blocks',
                'num_launched_supervisors',
                'num_launched_awcs',
                'cases_household',
                'cases_person',
                'cases_person_all',
                'cases_person_has_aadhaar',
                'cases_ccs_pregnant_all',
                'cases_ccs_lactating_all',
                'cases_child_health_all',
                'cases_person_adolescent_girls_11_14',
                'cases_person_adolescent_girls_15_18',
                'cases_person_adolescent_girls_11_14_all',
                'cases_person_adolescent_girls_15_18_all',
                'infra_infant_weighing_scale')

    def query(self):
        return """
        COPY (SELECT {columns} FROM {table} t LEFT JOIN awc_location awc on t.awc_id=awc.doc_id WHERE awc.state_id='{state_id}' AND t.month='{month}' and t.aggregation_level=5) TO STDOUT WITH CSV HEADER;
        """.format(
            columns=','.join(self.columns + self.location_columns),
            table=self.base_tablename,
            state_id=self.state_id,
            month=self.month
        )
<commit_msg>add name to child join condition<commit_after>from __future__ import absolute_import, unicode_literals

import tempfile

from corehq.apps.locations.models import SQLLocation
from corehq.apps.userreports.models import StaticDataSourceConfiguration, get_datasource_config
from corehq.apps.userreports.util import get_table_name

from custom.icds_reports.const import DASHBOARD_DOMAIN
from custom.icds_reports.models import CcsRecordMonthly, ChildHealthMonthly, AggAwc

class MBTHelper(object):

    def __init__(self, state_id, month):
        self.state_id = state_id
        self.month = month

    @property
    def domain(self):
        # Currently its only possible for one domain to have access to the ICDS dashboard per env
        return DASHBOARD_DOMAIN

    @property
    def columns(self):
        raise NotImplementedError

    @property
    def base_tablename(self):
        return self.base_class._meta.db_table

    @property
    def state_name(self):
        state = SQLLocation.objects.get(location_id=self.state_id)
        return state.name

    @property
    def output_file(self):
        temp_dir = tempfile.gettempdir()
        return '{}/{}_{}_{}.csv'.format(temp_dir, self.base_tablename, self.state_name, self.month)

    @property
    def location_columns(self):
        return ('awc.state_name', 'awc.district_name', 'awc.block_name', 'awc.awc_name', 'awc.awc_site_code')

    def query(self):
        return """
        COPY (SELECT {columns} FROM {table} t LEFT JOIN awc_location awc on t.awc_id=awc.doc_id WHERE awc.state_id='{state_id}' AND t.month='{month}') TO STDOUT WITH CSV HEADER;
        """.format(
            columns=','.join(self.columns + self.location_columns),
            table=self.base_tablename,
            state_id=self.state_id,
            month=self.month
        )

class CcsMbtHelper(MBTHelper):
    base_class = CcsRecordMonthly

    @property
    def columns(self):
        return ('t.awc_id',
                'case_id',
                'month',
                'age_in_months',
                'ccs_status',
                'open_in_month',
                'alive_in_month',
                'trimester',
                'num_rations_distributed',
                'thr_eligible',
                'tetanus_complete',
                'delivered_in_month',
                'anc1_received_at_delivery',
                'anc2_received_at_delivery',
                'anc3_received_at_delivery',
                'anc4_received_at_delivery',
                'registration_trimester_at_delivery',
                'using_ifa',
                'ifa_consumed_last_seven_days',
                'anemic_severe',
                'anemic_moderate',
                'anemic_normal',
                'anemic_unknown',
                'extra_meal',
                'resting_during_pregnancy',
                'bp_visited_in_month',
                'pnc_visited_in_month',
                'trimester_2',
                'trimester_3',
                'counsel_immediate_bf',
                'counsel_bp_vid',
                'counsel_preparation',
                'counsel_fp_vid',
                'counsel_immediate_conception',
                'counsel_accessible_postpartum_fp',
                'bp1_complete',
                'bp2_complete',
                'bp3_complete',
                'pnc_complete',
                'postnatal',
                'has_aadhar_id',
                'counsel_fp_methods',
                'pregnant',
                'pregnant_all',
                'lactating',
                'lactating_all',
                'institutional_delivery_in_month',
                'add',
                'anc_in_month',
                'caste',
                'disabled',
                'minority',
                'resident',
                'anc_weight',
                'anc_blood_pressure',
                'bp_sys',
                'bp_dia',
                'anc_hemoglobin',
                'bleeding',
                'swelling',
                'blurred_vision',
                'convulsions',
                'rupture',
                'anemia',
                'eating_extra',
                'resting',
                'immediate_breastfeeding',
                'edd',
                'delivery_nature',
                'is_ebf',
                'breastfed_at_birth',
                'anc_1',
                'anc_2',
                'anc_3',
                'anc_4',
                'tt_1',
                'tt_2',
                'valid_in_month',
                'mobile_number',
                'preg_order',
                'home_visit_date',
                'num_pnc_visits',
                'last_date_thr',
                'num_anc_complete',
                'opened_on',
                'valid_visits',
                'dob',
                'date_death')

class ChildHealthMbtHelper(MBTHelper):
    base_class = ChildHealthMonthly

    @property
    def person_case_ucr_tablename(self):
        doc_id = StaticDataSourceConfiguration.get_doc_id(self.domain, 'static-person_cases_v2')
        config, _ = get_datasource_config(doc_id, self.domain)
        return get_table_name(self.domain, config.table_id)

    @property
    def ccs_record_monthly_ucr_tablename(self):
        doc_id = StaticDataSourceConfiguration.get_doc_id(self.domain, 'static-ccs_record_cases_monthly_tableau_v2')
        config, _ = get_datasource_config(doc_id, self.domain)
        return get_table_name(self.domain, config.table_id)

    @property
    def columns(self):
        return ('t.awc_id',
                't.case_id',
                't.month',
                't.age_in_months',
                't.open_in_month',
                't.alive_in_month',
                't.wer_eligible',
                't.nutrition_status_last_recorded',
                't.current_month_nutrition_status',
                't.nutrition_status_weighed',
                't.num_rations_distributed',
                't.pse_eligible',
                't.pse_days_attended',
                't.born_in_month',
                't.low_birth_weight_born_in_month',
                't.bf_at_birth_born_in_month',
                't.ebf_eligible',
                't.ebf_in_month',
                't.ebf_not_breastfeeding_reason',
                't.ebf_drinking_liquid',
                't.ebf_eating',
                't.ebf_no_bf_no_milk',
                't.ebf_no_bf_pregnant_again',
                't.ebf_no_bf_child_too_old',
                't.ebf_no_bf_mother_sick',
                't.cf_eligible',
                't.cf_in_month',
                't.cf_diet_diversity',
                't.cf_diet_quantity',
                't.cf_handwashing',
                't.cf_demo',
                't.fully_immunized_eligible',
                't.fully_immunized_on_time',
                't.fully_immunized_late',
                't.counsel_ebf',
                't.counsel_adequate_bf',
                't.counsel_pediatric_ifa',
                't.counsel_comp_feeding_vid',
                't.counsel_increase_food_bf',
                't.counsel_manage_breast_problems',
                't.counsel_skin_to_skin',
                't.counsel_immediate_breastfeeding',
                't.recorded_weight',
                't.recorded_height',
                't.has_aadhar_id',
                't.thr_eligible',
                't.pnc_eligible',
                't.cf_initiation_in_month',
                't.cf_initiation_eligible',
                't.height_measured_in_month',
                't.current_month_stunting',
                't.stunting_last_recorded',
                't.wasting_last_recorded',
                't.current_month_wasting',
                't.valid_in_month',
                't.valid_all_registered_in_month',
                't.ebf_no_info_recorded',
                't.dob',
                't.sex',
                't.age_tranche',
                't.caste',
                't.disabled',
                't.minority',
                't.resident',
                't.immunization_in_month',
                't.days_ration_given_child',
                't.zscore_grading_hfa',
                't.zscore_grading_hfa_recorded_in_month',
                't.zscore_grading_wfh',
                't.zscore_grading_wfh_recorded_in_month',
                't.muac_grading',
                'ccs.case_id as ccs_record_case_id',
                't.date_death')

    def query(self):
        return """
        COPY (SELECT {columns} FROM {table} t
        LEFT JOIN awc_location awc on t.awc_id=awc.doc_id
        LEFT JOIN "{person_cases_ucr}" mother on mother.doc_id=t.mother_case_id
          AND awc.state_id = mother.state_id
          AND lower(substring(mother.state_id, '.{{3}}$'::text)) = '{state_id_last_3}'
        LEFT JOIN "ccs_record_monthly_{month}" ccs on ccs.person_case_id=mother.doc_id AND ccs.add=t.dob AND ccs.child_name=t.person_name
        WHERE awc.state_id='{state_id}' AND t.month='{month}')
        TO STDOUT WITH CSV HEADER;
        """.format(
            columns=','.join(self.columns + self.location_columns),
            table=self.base_tablename,
            state_id=self.state_id,
            month=self.month,
            person_cases_ucr=self.person_case_ucr_tablename,
            ccs_record_monthly=self.ccs_record_monthly_ucr_tablename,
            state_id_last_3=self.state_id[-3:]
        )

class AwcMbtHelper(MBTHelper):
    base_class = AggAwc

    @property
    def columns(self):
        return ('t.state_id',
                't.district_id',
                't.block_id',
                't.supervisor_id',
                't.awc_id',
                'month',
                'num_awcs',
                'awc_days_open',
                'total_eligible_children',
                'total_attended_children',
                'pse_avg_attendance_percent',
                'pse_full',
                'pse_partial',
                'pse_non',
                'pse_score',
                'awc_days_provided_breakfast',
                'awc_days_provided_hotmeal',
                'awc_days_provided_thr',
                'awc_days_provided_pse',
                'awc_not_open_holiday',
                'awc_not_open_festival',
                'awc_not_open_no_help',
                'awc_not_open_department_work',
                'awc_not_open_other',
                'awc_num_open',
                'awc_not_open_no_data',
                'wer_weighed',
                'wer_eligible',
                'wer_score',
                'thr_eligible_child',
                'thr_rations_21_plus_distributed_child',
                'thr_eligible_ccs',
                'thr_rations_21_plus_distributed_ccs',
                'thr_score',
                'awc_score',
                'num_awc_rank_functional',
                'num_awc_rank_semi',
                'num_awc_rank_non',
                'cases_ccs_pregnant',
                'cases_ccs_lactating',
                'cases_child_health',
                'usage_num_pse',
                'usage_num_gmp',
                'usage_num_thr',
                'usage_num_home_visit',
                'usage_num_bp_tri1',
                'usage_num_bp_tri2',
                'usage_num_bp_tri3',
                'usage_num_pnc',
                'usage_num_ebf',
                'usage_num_cf',
                'usage_num_delivery',
                'usage_num_due_list_ccs',
                'usage_num_due_list_child_health',
                'usage_awc_num_active',
                'usage_time_pse',
                'usage_time_gmp',
                'usage_time_bp',
                'usage_time_pnc',
                'usage_time_ebf',
                'usage_time_cf',
                'usage_time_of_day_pse',
                'usage_time_of_day_home_visit',
                'vhnd_immunization',
                'vhnd_anc',
                'vhnd_gmp',
                'vhnd_num_pregnancy',
                'vhnd_num_lactating',
                'vhnd_num_mothers_6_12',
                'vhnd_num_mothers_12',
                'vhnd_num_fathers',
                'ls_supervision_visit',
                'ls_num_supervised',
                'ls_awc_location_long',
                'ls_awc_location_lat',
                'ls_awc_present',
                'ls_awc_open',
                'ls_awc_not_open_aww_not_available',
                'ls_awc_not_open_closed_early',
                'ls_awc_not_open_holiday',
                'ls_awc_not_open_unknown',
                'ls_awc_not_open_other',
                'infra_last_update_date',
                'infra_type_of_building',
                'infra_type_of_building_pucca',
                'infra_type_of_building_semi_pucca',
                'infra_type_of_building_kuccha',
                'infra_type_of_building_partial_covered_space',
                'infra_clean_water',
                'infra_functional_toilet',
                'infra_baby_weighing_scale',
                'infra_flat_weighing_scale',
                'infra_adult_weighing_scale',
                'infra_cooking_utensils',
                'infra_medicine_kits',
                'infra_adequate_space_pse',
                'cases_person_beneficiary',
                'cases_person_referred',
                'awc_days_pse_conducted',
                'num_awc_infra_last_update',
                'cases_person_has_aadhaar_v2',
                'cases_person_beneficiary_v2',
                'electricity_awc',
                'infantometer',
                'stadiometer',
                'num_anc_visits',
                'num_children_immunized',
                'usage_num_hh_reg',
                'usage_num_add_person',
                'usage_num_add_pregnancy',
                'is_launched',
                'training_phase',
                'trained_phase_1',
                'trained_phase_2',
                'trained_phase_3',
                'trained_phase_4',
                't.aggregation_level',
                'num_launched_states',
                'num_launched_districts',
                'num_launched_blocks',
                'num_launched_supervisors',
                'num_launched_awcs',
                'cases_household',
                'cases_person',
                'cases_person_all',
                'cases_person_has_aadhaar',
                'cases_ccs_pregnant_all',
                'cases_ccs_lactating_all',
                'cases_child_health_all',
                'cases_person_adolescent_girls_11_14',
                'cases_person_adolescent_girls_15_18',
                'cases_person_adolescent_girls_11_14_all',
                'cases_person_adolescent_girls_15_18_all',
                'infra_infant_weighing_scale')

    def query(self):
        return """
        COPY (SELECT {columns} FROM {table} t LEFT JOIN awc_location awc on t.awc_id=awc.doc_id WHERE awc.state_id='{state_id}' AND t.month='{month}' and t.aggregation_level=5) TO STDOUT WITH CSV HEADER;
        """.format(
            columns=','.join(self.columns + self.location_columns),
            table=self.base_tablename,
            state_id=self.state_id,
            month=self.month
        )
<|endoftext|>