Python ncf.get_all_generic_methods_metadata函数代码示例

  def test_get_all_generic_methods_metadata_with_arg(self):
    """get_all_generic_methods_metadata should return a list of all generic_methods with all defined metadata tags"""
    alternative_path = os.path.dirname(os.path.realpath(__file__)) + "/test_methods"
    metadata = ncf.get_all_generic_methods_metadata(alternative_path)["data"]

    number_generic_methods = len(ncf.get_all_generic_methods_filenames(alternative_path))
    self.assertEquals(number_generic_methods, len(metadata))

def get_technique_expected_reports(technique_metadata):
    """Generates technique expected reports from technique metadata"""
    # Get all generic methods
    generic_methods = ncf.get_all_generic_methods_metadata()

    # Content start with a header
    content = [
        """# This file contains one line per report expected by Rudder from this technique
# Format: technique_name;;class_prefix_${key};;@@[email protected]@;;component name;;component key"""
    ]

    technique_name = technique_metadata["bundle_name"]
    for method_call in technique_metadata["method_calls"]:
        # Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
        if method_call["method_name"].startswith("_"):
            continue

        method_name = method_call["method_name"]
        generic_method = generic_methods[method_name]

        component = generic_method["name"]
        key_value = method_call["args"][generic_method["class_parameter_id"] - 1]
        class_prefix = generic_method["class_prefix"] + "_" + key_value

        line = technique_name + ";;" + class_prefix + ";;@@[email protected]@;;" + component + ";;" + key_value

        content.append(line)

    # Join all lines + last line
    result = "\n".join(content) + "\n"
    return result

def get_technique_metadata_xml(technique_metadata):
  """Get metadata xml for a technique as string"""

  # Get all generic methods
  generic_methods = ncf.get_all_generic_methods_metadata()

  content = []
  content.append('<TECHNIQUE name="'+technique_metadata['name']+'">')
  content.append('  <DESCRIPTION>'+technique_metadata['description']+'</DESCRIPTION>')
  content.append('  <BUNDLES>')
  content.append('    <NAME>'+ technique_metadata['bundle_name'] + '</NAME>')
  content.append('  </BUNDLES>')
  content.append('  <SECTIONS>')

  method_calls = technique_metadata["method_calls"]  

  # Get all method call, with no duplicate values
  methods_name = set()
  for method_call in method_calls:
    method_name = methods_name.add(method_call['method_name'])

  # For each method used, create a section containing all calls to that method
  section_list = []
  for method_name in methods_name:

    try:
      generic_method = generic_methods[method_name]
    except Exception, e:
      print "Error: The method '" + method_name + "' does not exist. Aborting Technique creation..."

    # Filter all method calls to get only those about that method
    filter_method_calls = [x for x in method_calls if x["method_name"] == method_name]
    # Generare xml for that section
    section = generate_section_xml(filter_method_calls, generic_method)
    section_list.extend(section)

def get_technique_metadata_xml(technique_metadata, include_rudder_reporting = False):
  """Get metadata xml for a technique as string"""

  # Get all generic methods
  generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']

  content = []
  content.append('<TECHNIQUE name="'+technique_metadata['name']+'">')
  content.append('  <DESCRIPTION>'+technique_metadata['description']+'</DESCRIPTION>')
  content.append('  <BUNDLES>')
  content.append('    <NAME>'+ technique_metadata['bundle_name'] + '</NAME>')
  if include_rudder_reporting:
    content.append('    <NAME>'+ technique_metadata['bundle_name'] + '_rudder_reporting</NAME>')
  content.append('  </BUNDLES>')

  if include_rudder_reporting:
    content.append('  <TMLS>')
    content.append('    <TML name="rudder_reporting"/>')
    content.append('  </TMLS>')

  content.append('  <SECTIONS>')

  method_calls = technique_metadata["method_calls"]  

  # Get all method call, with no duplicate values
  methods_name = set()
  for method_call in method_calls:
    # Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
    if method_call['method_name'].startswith("_"):
      continue

    method_name = methods_name.add(method_call['method_name'])

  # For each method used, create a section containing all calls to that method
  section_list = []
  for method_name in methods_name:

    try:
      generic_method = generic_methods[method_name]
    except Exception as e:
      sys.stderr.write("Error: The method '" + method_name + "' does not exist. Aborting Technique creation..." + "\n")
      sys.stderr.write(traceback.format_exc())
      exit(1)

    # Filter all method calls to get only those about that method
    filter_method_calls = [x for x in method_calls if x["method_name"] == method_name]
    # Generare xml for that section
    section = generate_section_xml(filter_method_calls, generic_method)
    section_list.extend(section)

  content.extend(section_list)
  content.append('  </SECTIONS>')
  content.append('</TECHNIQUE>')

  # Join all lines with \n to get a pretty xml
  result =  '\n'.join(content)+"\n"

  return result

def get_generic_methods():
  """Get all generic methods from ncf folder passed as parameter, or default ncf folder if not defined"""
  # We need to get path from url params, if not present put "" as default value
  if "path" in request.args:
    path = request.args['path']
  else:
    path = ""

  generic_methods = ncf.get_all_generic_methods_metadata(alt_path = path)
  resp = jsonify( generic_methods )
  return resp

def get_generic_methods():
  """Get all generic methods from ncf folder passed as parameter, or default ncf folder if not defined"""
  try:
    # We need to get path from url params, if not present put "" as default value
    path = get_path_from_args(request)
  
    generic_methods = ncf.get_all_generic_methods_metadata(alt_path = path)
    resp = jsonify( generic_methods )
    return resp
  except Exception as e:
    return format_error(e, "generic methods fetching", 500)

def write_technique_for_rudder(root_path, technique):
  """ From a technique, generate all files needed for Rudder in specified path"""

  path = get_path_for_technique(root_path,technique)
  if not os.path.exists(path):
    os.makedirs(path)
  # We don't need to create rudder_reporting.st if all class context are any
  generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']['generic_methods']
  include_rudder_reporting = not all(method_call['class_context'] == 'any' and 
                                     "cfengine-community" in generic_methods[method_call['method_name']]["agent_support"] for method_call in technique["method_calls"])
  if include_rudder_reporting:
    write_rudder_reporting_file(path,technique)

def generate_rudder_reporting(technique):
  """Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
  # Get all generic methods
  generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')

  content = []
  content.append('bundle agent '+ technique['bundle_name']+'_rudder_reporting')
  content.append('{')
  content.append('  methods:')

  # Handle method calls
  technique_name = technique['bundle_name']
  # Filter calls so we only have those who are not using a any class_context
  filter_calls = [ method_call for method_call in technique["method_calls"] if method_call['class_context'] != "any" ]

  for method_call in filter_calls:

    method_name = method_call['method_name']
    generic_method = generic_methods[method_name]

    key_value = method_call["args"][generic_method["class_parameter_id"]-1]
    key_value_canonified = re.sub("[^\$\{\}\w](?![^{}]+})|\$(?!{)", "_", key_value)


    class_prefix = generic_method["class_prefix"]+"_"+key_value_canonified

    # Always add an empty line for readability
    content.append('')

    if not "$" in method_call['class_context']:
      content.append('    !('+method_call['class_context']+')::')
      content.append('      "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
      content.append('      "dummy_report" usebundle => logger_rudder("Not applicable", "'+class_prefix+'");')

    else:
      content.append('      "dummy_report" usebundle => _classes_noop("'+class_prefix+'"),')
      content.append('                    ifvarclass => "!('+method_call['class_context']+')";')
      content.append('      "dummy_report" usebundle => logger_rudder("Not applicable", "'+class_prefix+'"),')
      content.append('                    ifvarclass => "!('+method_call['class_context']+')";')




  content.append('}')

  # Join all lines with \n to get a pretty CFEngine file
  result =  '\n'.join(content)+"\n"

  return result

def get_technique_metadata_xml(technique_metadata, include_rudder_reporting=False):
    """Get metadata xml for a technique as string"""

    # Get all generic methods
    generic_methods = ncf.get_all_generic_methods_metadata(alt_path="/var/rudder/configuration-repository/ncf")

    content = []
    content.append('<TECHNIQUE name="' + technique_metadata["name"] + '">')
    content.append("  <DESCRIPTION>" + technique_metadata["description"] + "</DESCRIPTION>")
    content.append("  <BUNDLES>")
    content.append("    <NAME>" + technique_metadata["bundle_name"] + "</NAME>")
    if include_rudder_reporting:
        content.append("    <NAME>" + technique_metadata["bundle_name"] + "_rudder_reporting</NAME>")
    content.append("  </BUNDLES>")

    if include_rudder_reporting:
        content.append("  <TMLS>")
        content.append('    <TML name="rudder_reporting"/>')
        content.append("  </TMLS>")

    content.append("  <SECTIONS>")

    method_calls = technique_metadata["method_calls"]

    # Get all method call, with no duplicate values
    methods_name = set()
    for method_call in method_calls:
        # Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
        if method_call["method_name"].startswith("_"):
            continue

        method_name = methods_name.add(method_call["method_name"])

    # For each method used, create a section containing all calls to that method
    section_list = []
    for method_name in methods_name:

        try:
            generic_method = generic_methods[method_name]
        except Exception, e:
            print "Error: The method '" + method_name + "' does not exist. Aborting Technique creation..."

        # Filter all method calls to get only those about that method
        filter_method_calls = [x for x in method_calls if x["method_name"] == method_name]
        # Generare xml for that section
        section = generate_section_xml(filter_method_calls, generic_method)
        section_list.extend(section)

  def setUp(self):
    self.maxDiff = None
    
    self.test_technique_file = os.path.realpath('test_technique.cf')
    with open(self.test_technique_file) as fd:
      self.technique_content = fd.read()
    self.all_methods = ncf.get_all_generic_methods_metadata()["data"]["generic_methods"]  
    self.technique_metadata = ncf.parse_technique_metadata(self.technique_content)['result']
    method_calls = ncf.parse_technique_methods(self.test_technique_file, self.all_methods)
    self.technique_metadata['method_calls'] = method_calls

    self.test_reporting = os.path.realpath('test_technique_reporting.cf')
    with open(self.test_reporting) as fd:
      self.reporting_content = fd.read()
    self.reporting_metadata = ncf.parse_technique_metadata(self.reporting_content)['result']
    reporting_method_calls = ncf.parse_technique_methods(self.test_reporting, self.all_methods)
    self.reporting_metadata['method_calls'] = reporting_method_calls

    self.test_reporting_with_var = os.path.realpath('test_technique_with_variable.cf')
    with open(self.test_reporting_with_var) as fd:
      self.reporting_with_var_content = fd.read()
    self.reporting_with_var_metadata = ncf.parse_technique_metadata(self.reporting_with_var_content)['result']
    reporting_with_var_method_calls = ncf.parse_technique_methods(self.test_reporting_with_var, self.all_methods)
    self.reporting_with_var_metadata['method_calls'] = reporting_with_var_method_calls


    self.test_any_technique = os.path.realpath('test_technique_any.cf')
    with open(self.test_any_technique) as fd:
      self.any_technique_content = fd.read()
    self.any_technique_metadata = ncf.parse_technique_metadata(self.any_technique_content)['result']
    any_technique_method_calls = ncf.parse_technique_methods(self.test_any_technique, self.all_methods)
    self.any_technique_metadata['method_calls'] = any_technique_method_calls

    self.rudder_reporting_file = os.path.realpath('test_technique_rudder_reporting.cf')
    with open(self.rudder_reporting_file) as fd:
      self.rudder_reporting_content = fd.read()

    # Testing Techniques with quote
    self.test_technique_with_quote_file = os.path.realpath('test_technique_with_quote.cf')
    with open(self.test_technique_with_quote_file) as fd:
      self.technique_with_quote_content = fd.read()
    self.technique_with_quote_metadata = ncf.parse_technique_metadata(self.technique_with_quote_content)['result']
    method_with_quote_calls = ncf.parse_technique_methods(self.test_technique_with_quote_file, self.all_methods)
    self.technique_with_quote_metadata['method_calls'] = method_with_quote_calls

def generate_rudder_reporting(technique):
    """Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
    # Get all generic methods
    generic_methods = ncf.get_all_generic_methods_metadata()

    content = []
    content.append("bundle agent " + technique["bundle_name"] + "_rudder_reporting")
    content.append("{")
    content.append("  methods:")

    # Handle method calls
    technique_name = technique["bundle_name"]
    # Filter calls so we only have those who are not using a any class_context
    filter_calls = [method_call for method_call in technique["method_calls"] if method_call["class_context"] != "any"]

    for method_call in filter_calls:

        content.append("    !(" + method_call["class_context"] + ")::")

        method_name = method_call["method_name"]
        generic_method = generic_methods[method_name]

        key_value = method_call["args"][generic_method["class_parameter_id"] - 1]
        key_value_canonified = re.sub("[^a-zA-Z0-9_]", "_", key_value)

        class_prefix = generic_method["class_prefix"] + "_" + key_value_canonified

        content.append('      "dummy_report" usebundle => _classes_noop("' + class_prefix + '");')
        content.append('      "dummy_report" usebundle => logger_rudder("Not applicable", "' + class_prefix + '");')

    content.append("}")

    # Join all lines with \n to get a pretty CFEngine file
    result = "\n".join(content) + "\n"

    return result

  def test_get_all_generic_methods_metadata(self):
    """get_all_generic_methods_metadata should return a list of all generic_methods with all defined metadata tags"""
    metadata = ncf.get_all_generic_methods_metadata()["data"]

    number_generic_methods = len(ncf.get_all_generic_methods_filenames())
    self.assertEquals(number_generic_methods, len(metadata))

  
  cfengine_versions = list(min_version.keys())
  cfengine_versions.sort(key=StrictVersion)
  
  for cfengine_version in cfengine_versions:
    if StrictVersion(canonified_version) <= StrictVersion(cfengine_version):
      return min_version[cfengine_version]
  return False


if __name__ == '__main__':
  # Get CFEngine <=> Rudder versions
  versions = get_min_versions()

  # Get all generic methods
  generic_methods = ncf.get_all_generic_methods_metadata()["data"]["generic_methods"]
  
  categories = {}
  for method_name in sorted(generic_methods.keys()):
    category_name = method_name.split('_',1)[0]
    generic_method = generic_methods[method_name]
    if (category_name in categories):
      categories[category_name].append(generic_method)
    else:
      categories[category_name] = [generic_method]

  content = []

  for category in sorted(categories.keys()):
    content.append("\n*****\n")
    content.append('\n### '+category.title())

def generate_rudder_reporting(technique):
  """Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
  # Get all generic methods
  generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']['generic_methods']

  args = ""
  bundle_param = ""
  if len(technique["parameter"]) > 0:
    bundle_param = "("+", ".join([ncf.canonify(param["name"]) for param in technique["parameter"] ])+")"
    args = ", ".join('"${' +  ncf.canonify(param["name"]) + '}"' for param in technique["parameter"] )

  bundle_name = technique['bundle_name']+'_rudder_reporting'

  content = []
  content.append('bundle agent '+ technique['bundle_name']+'_rudder_reporting'+ bundle_param)
  content.append('{')
  content.append('  vars:')
  content.append('    "args"               slist => { '+ args +' };')
  content.append('    "report_param"      string => join("_", args);')
  content.append('    "full_class_prefix" string => canonify("' + bundle_name + '_${report_param}");')
  content.append('    "class_prefix"      string => string_head("${full_class_prefix}", "1000");')
  content.append('')
  content.append('  methods:')

  report_unique_id = 0

  for method_call in technique["method_calls"]:

    method_name = method_call['method_name']
    if method_call['method_name'].startswith("_"):
      continue
    generic_method = generic_methods[method_name]

    key_value = ncf.get_key_value(method_call, generic_method)

    class_prefix = '${class_prefix}_'+ncf.get_class_prefix(key_value, generic_method)
    method_reporting = '"dummy_report_' + str(report_unique_id) + '" usebundle => ' + ncf.generate_reporting_context(generic_method, method_call) 
    report_unique_id += 1
    class_parameter  = ncf.generate_reporting_class_parameter(generic_method, method_call)

    if not "cfengine-community" in generic_method["agent_support"]:

      message = "'"+generic_method["name"]+"' method is not available on cfengine based agent, skip"
      logger_call = get_logger_call(message, class_prefix, class_parameter)

      content.append('    any::')
      content.append('    "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
      content.append('    ' + method_reporting + ';')
      content.append('    ' + logger_call+';')

    elif method_call['class_context'] != "any":
      # escape double quote in key value
      regex_quote = re.compile(r'(?<!\\)"', flags=re.UNICODE )
      escaped_key_value = regex_quote.sub('\\"', key_value)

      message = generic_method['name'] + ' ' + escaped_key_value + ' if ' + method_call['class_context']
      logger_rudder_call = get_logger_call(message, class_prefix, class_parameter)

      # Always add an empty line for readability
      content.append('')

      if not "$" in method_call['class_context']:
        content.append('    !('+method_call['class_context']+')::')
        content.append('      "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
        content.append('      ' + method_reporting + ';')
        content.append('      ' + logger_rudder_call + ';')

      else:
        class_context = ncf.canonify_class_context(method_call['class_context'])

        content.append('      "dummy_report" usebundle => _classes_noop("'+class_prefix+'"),')
        content.append('                    ifvarclass => concat("!('+class_context+')");')
        content.append('      ' + method_reporting + ',')
        content.append('                    ifvarclass => concat("!('+class_context+')");')
        content.append('      ' + logger_rudder_call + ',')
        content.append('                    ifvarclass => concat("!('+class_context+')");')

  content.append('}')

  # Join all lines with \n to get a pretty CFEngine file
  result =  '\n'.join(content)+"\n"

  return result

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Usage: ./ncf_doc.py
#
# This is a Python module to generate documentation from generic methods in ncf

import ncf 

if __name__ == '__main__':

  # Get all generic methods
  generic_methods = ncf.get_all_generic_methods_metadata()
  
  
  categories = {}
  for method_name in sorted(generic_methods.iterkeys()):
    category_name = method_name.split('_',1)[0]
    generic_method = generic_methods[method_name]
    if (category_name in categories):
      categories[category_name].append(generic_method)
    else:
      categories[category_name] = [generic_method]
    


  content = []
  html_content = []

  content.append("Title: Reference")
  content.append("slugs: reference")