cli.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from argparse import ArgumentParser, RawDescriptionHelpFormatter
  15. import yaml
  16. import re
  17. from ppdet.core.workspace import get_registered_modules, dump_value
  18. __all__ = ['ColorTTY', 'ArgsParser']
  19. class ColorTTY(object):
  20. def __init__(self):
  21. super(ColorTTY, self).__init__()
  22. self.colors = ['red', 'green', 'yellow', 'blue', 'magenta', 'cyan']
  23. def __getattr__(self, attr):
  24. if attr in self.colors:
  25. color = self.colors.index(attr) + 31
  26. def color_message(message):
  27. return "[{}m{}".format(color, message)
  28. setattr(self, attr, color_message)
  29. return color_message
  30. def bold(self, message):
  31. return self.with_code('01', message)
  32. def with_code(self, code, message):
  33. return "[{}m{}".format(code, message)
  34. class ArgsParser(ArgumentParser):
  35. def __init__(self):
  36. super(ArgsParser, self).__init__(
  37. formatter_class=RawDescriptionHelpFormatter)
  38. self.add_argument("-c", "--config", help="configuration file to use")
  39. self.add_argument(
  40. "-o", "--opt", nargs='*', help="set configuration options")
  41. def parse_args(self, argv=None):
  42. args = super(ArgsParser, self).parse_args(argv)
  43. assert args.config is not None, \
  44. "Please specify --config=configure_file_path."
  45. args.opt = self._parse_opt(args.opt)
  46. return args
  47. def _parse_opt(self, opts):
  48. config = {}
  49. if not opts:
  50. return config
  51. for s in opts:
  52. s = s.strip()
  53. k, v = s.split('=', 1)
  54. if '.' not in k:
  55. config[k] = yaml.load(v, Loader=yaml.Loader)
  56. else:
  57. keys = k.split('.')
  58. if keys[0] not in config:
  59. config[keys[0]] = {}
  60. cur = config[keys[0]]
  61. for idx, key in enumerate(keys[1:]):
  62. if idx == len(keys) - 2:
  63. cur[key] = yaml.load(v, Loader=yaml.Loader)
  64. else:
  65. cur[key] = {}
  66. cur = cur[key]
  67. return config
  68. def print_total_cfg(config):
  69. modules = get_registered_modules()
  70. color_tty = ColorTTY()
  71. green = '___{}___'.format(color_tty.colors.index('green') + 31)
  72. styled = {}
  73. for key in config.keys():
  74. if not config[key]: # empty schema
  75. continue
  76. if key not in modules and not hasattr(config[key], '__dict__'):
  77. styled[key] = config[key]
  78. continue
  79. elif key in modules:
  80. module = modules[key]
  81. else:
  82. type_name = type(config[key]).__name__
  83. if type_name in modules:
  84. module = modules[type_name].copy()
  85. module.update({
  86. k: v
  87. for k, v in config[key].__dict__.items()
  88. if k in module.schema
  89. })
  90. key += " ({})".format(type_name)
  91. default = module.find_default_keys()
  92. missing = module.find_missing_keys()
  93. mismatch = module.find_mismatch_keys()
  94. extra = module.find_extra_keys()
  95. dep_missing = []
  96. for dep in module.inject:
  97. if isinstance(module[dep], str) and module[dep] != '<value>':
  98. if module[dep] not in modules: # not a valid module
  99. dep_missing.append(dep)
  100. else:
  101. dep_mod = modules[module[dep]]
  102. # empty dict but mandatory
  103. if not dep_mod and dep_mod.mandatory():
  104. dep_missing.append(dep)
  105. override = list(
  106. set(module.keys()) - set(default) - set(extra) - set(dep_missing))
  107. replacement = {}
  108. for name in set(override + default + extra + mismatch + missing):
  109. new_name = name
  110. if name in missing:
  111. value = "<missing>"
  112. else:
  113. value = module[name]
  114. if name in extra:
  115. value = dump_value(value) + " <extraneous>"
  116. elif name in mismatch:
  117. value = dump_value(value) + " <type mismatch>"
  118. elif name in dep_missing:
  119. value = dump_value(value) + " <module config missing>"
  120. elif name in override and value != '<missing>':
  121. mark = green
  122. new_name = mark + name
  123. replacement[new_name] = value
  124. styled[key] = replacement
  125. buffer = yaml.dump(styled, default_flow_style=False, default_style='')
  126. buffer = (re.sub(r"<missing>", r"[31m<missing>[0m", buffer))
  127. buffer = (re.sub(r"<extraneous>", r"[33m<extraneous>[0m", buffer))
  128. buffer = (re.sub(r"<type mismatch>", r"[31m<type mismatch>[0m", buffer))
  129. buffer = (re.sub(r"<module config missing>",
  130. r"[31m<module config missing>[0m", buffer))
  131. buffer = re.sub(r"___(\d+)___(.*?):", r"[\1m\2[0m:", buffer)
  132. print(buffer)