Allow comment in YAML output files (#1069)

* Allow comments to be associated with map assignments

* Use new comment mechanism for model comparer

* Remove comment key mechanism and special processing for JSON

Author: rakillen

core/src/main/java/oracle/weblogic/deploy/util/CommentMap.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+ */
+package oracle.weblogic.deploy.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Maps dictionary keys to lists of comments, for use in output formatting.
+ */
+public class CommentMap {
+    public static final String BLANK_LINE_KEY = "__BLANK_LINE__";
+
+    private final Map<String, List<String>> commentMap = new HashMap<>();
+
+    public void addBlankLine(String key) {
+        getOrCreateComments(key).add(BLANK_LINE_KEY);
+    }
+
+    public void addComment(String key, String comment) {
+        getOrCreateComments(key).add(comment);
+    }
+
+    public List<String> getComments(String key) {
+        List<String> comments = commentMap.get(key);
+        return (comments == null) ? Collections.<String>emptyList() : comments;
+    }
+
+    private List<String> getOrCreateComments(String key) {
+        List<String> comments = commentMap.get(key);
+        if(comments == null) {
+            comments = new ArrayList<>();
+            commentMap.put(key, comments);
+        }
+        return comments;
+    }
+}

core/src/main/java/oracle/weblogic/deploy/util/OrderedMap.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+ */
+package oracle.weblogic.deploy.util;
+
+import java.util.LinkedHashMap;
+
+/**
+ * A map that maintains the order of its assignments,
+ * and has a mapping of element keys to a list of comments.
+ */
+public class OrderedMap extends LinkedHashMap<String, Object> {
+    private CommentMap commentMap = new CommentMap();
+
+    public OrderedMap() {
+        super();
+    }
+
+    public CommentMap getCommentMap() {
+        return commentMap;
+    }
+
+    public void setCommentMap(CommentMap commentMap) {
+        this.commentMap = commentMap;
+    }
+}

core/src/main/java/oracle/weblogic/deploy/util/PyOrderedDict.java

@@ -37,6 +37,8 @@ public final class PyOrderedDict extends PyDictionary implements Iterable<PyObje
 
     private final LinkedHashMap<PyObject, PyObject> linkedHashMap;
 
+    private final CommentMap commentMap = new CommentMap();
+
     /**
      * The no-args constructor.
      */
@@ -477,6 +479,14 @@ public PyList getValues() {
         return new PyList(v.toArray(new PyObject[0]));
     }
 
+    public CommentMap getCommentMap() {
+        return commentMap;
+    }
+
+    public void addComment(String key, String comment) {
+        commentMap.addComment(key, comment);
+    }
+
     // private methods
 
     private static PyObject dictFromKeys(PyType type, PyObject keys, PyObject value) {

core/src/main/java/oracle/weblogic/deploy/yaml/AbstractYamlTranslator.java

@@ -92,7 +92,8 @@ protected void dumpInternal(Map<String, Object> data, Writer outputWriter) throw
 
         if (outputWriter != null) {
             DumperOptions dumperOptions = getDefaultDumperOptions();
-            Yaml yaml = new Yaml(dumperOptions);
+            YamlRepresenter representer = new YamlRepresenter();
+            Yaml yaml = new Yaml(representer, dumperOptions);
 
             try {
                 yaml.dump(replaceNoneInMap(data), outputWriter);

core/src/main/java/oracle/weblogic/deploy/yaml/YamlRepresenter.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+ */
+package oracle.weblogic.deploy.yaml;
+
+import oracle.weblogic.deploy.util.CommentMap;
+import oracle.weblogic.deploy.util.OrderedMap;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.comments.CommentLine;
+import org.yaml.snakeyaml.comments.CommentType;
+import org.yaml.snakeyaml.nodes.MappingNode;
+import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.NodeTuple;
+import org.yaml.snakeyaml.nodes.ScalarNode;
+import org.yaml.snakeyaml.nodes.Tag;
+import org.yaml.snakeyaml.representer.Representer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static oracle.weblogic.deploy.util.CommentMap.BLANK_LINE_KEY;
+
+/**
+ * Attach comments from mappings to the associated elements.
+ */
+public class YamlRepresenter extends Representer {
+
+  @Override
+  protected Node representMapping(Tag tag, Map<?, ?> mapping, DumperOptions.FlowStyle flowStyle) {
+    MappingNode node = (MappingNode) super.representMapping(tag, mapping, flowStyle);
+
+    if(mapping instanceof OrderedMap) {
+      for (NodeTuple tuple : node.getValue()) {
+        Object keyNode = tuple.getKeyNode();
+        if(keyNode instanceof ScalarNode) {
+          CommentMap commentMap = ((OrderedMap) mapping).getCommentMap();
+          String key = ((ScalarNode) keyNode).getValue();
+          List<String> comments = commentMap.getComments(key);
+          if(!comments.isEmpty()) {
+            List<CommentLine> tupleComments = new ArrayList<>();
+            for (String comment: comments) {
+              if(BLANK_LINE_KEY.equals(comment)) {
+                tupleComments.add(new CommentLine(null, null, "", CommentType.BLANK_LINE));
+              } else {
+                tupleComments.add(new CommentLine(null, null, " " + comment, CommentType.BLOCK));
+              }
+            }
+            tuple.getKeyNode().setBlockComments(tupleComments);
+          }
+        }
+      }
+    }
+
+    return node;
+  }
+}

core/src/main/python/wlsdeploy/json/json_translator.py

@@ -20,8 +20,6 @@
 from wlsdeploy.logging.platform_logger import PlatformLogger
 import wlsdeploy.exception.exception_helper as exception_helper
 
-# Unlike with yaml files, JSON files do not allow comments. remove from file
-COMMENT_MATCH = '# - '
 
 class JsonToPython(object):
     """
@@ -170,18 +168,15 @@ def _write_dictionary_to_json_file(self, dictionary, writer, indent=''):
 
         indent += self._indent_unit
         for key, value in dictionary.iteritems():
-            if isinstance(key, basestring) and key.startswith(COMMENT_MATCH):
-                self._logger.finer('WLSDPLY-01714', key, class_name=self._class_name, method_name=_method_name)
+            writer.println(end_line)
+            end_line = ','
+            writer.write(indent + '"' + _escape_text(key) + '" : ')
+            if isinstance(value, dict):
+                self._write_dictionary_to_json_file(value, writer, indent)
+            elif isinstance(value, list):
+                self._write_list_to_json_file(value, writer, indent)
             else:
-                writer.println(end_line)
-                end_line = ','
-                writer.write(indent + '"' + _escape_text(key) + '" : ')
-                if isinstance(value, dict):
-                    self._write_dictionary_to_json_file(value, writer, indent)
-                elif isinstance(value, list):
-                    self._write_list_to_json_file(value, writer, indent)
-                else:
-                    writer.write(_format_json_value(value))
+                writer.write(_format_json_value(value))
         writer.println()
         writer.write(end_indent + _end_dict)
 

core/src/main/python/wlsdeploy/tool/compare/model_comparer.py

@@ -14,7 +14,6 @@
 from wlsdeploy.aliases.model_constants import APPLICATION
 from wlsdeploy.aliases.model_constants import KUBERNETES
 from wlsdeploy.aliases.model_constants import LIBRARY
-from wlsdeploy.json.json_translator import COMMENT_MATCH
 from wlsdeploy.logging.platform_logger import PlatformLogger
 from wlsdeploy.util import dictionary_utils
 from wlsdeploy.util import model_helper
@@ -129,8 +128,9 @@ def _compare_security_folders(self, current_folder, past_folder, location, attri
             self._messages.add(('WLSDPLY-05716', location.get_folder_path()))
             comment = "Replace entire Security Provider section "
             new_folder = PyOrderedDict()
-            _add_comment(comment, new_folder)
             new_folder.update(current_folder)
+            if new_folder:
+                new_folder.addComment(new_folder.keys()[0], comment)
             return new_folder
         return PyOrderedDict()
 
@@ -359,7 +359,7 @@ def _compare_attribute(self, current_value, past_value, location, key, change_fo
                 current_text = ','.join(current_list)
                 previous_text = ','.join(previous_list)
                 comment = key + ": '" + previous_text + "' -> '" + current_text + "'"
-                _add_comment(comment, change_folder)
+                change_folder.addComment(key, comment)
                 change_folder[key] = ','.join(change_list)
 
             elif attribute_type == PROPERTIES:
@@ -368,7 +368,7 @@ def _compare_attribute(self, current_value, past_value, location, key, change_fo
             else:
                 if not isinstance(past_value, dict):
                     comment = key + ": '" + str(past_value) + "'"
-                    _add_comment(comment, change_folder)
+                    change_folder.addComment(key, comment)
                 change_folder[key] = current_value
 
     def _compare_properties(self, current_value, past_value, key, change_folder):
@@ -387,7 +387,7 @@ def _compare_properties(self, current_value, past_value, key, change_folder):
                 past_property_value = past_value[property_key]
                 if past_property_value != current_property_value:
                     comment = property_key + ": '" + str(past_property_value) + "'"
-                    _add_comment(comment, property_dict)
+                    property_dict.addComment(property_key, comment)
                     property_dict[property_key] = current_property_value
             else:
                 property_dict[property_key] = current_property_value
@@ -436,14 +436,3 @@ def _finalize_folder(self, current_folder, past_folder, change_folder, location)
                         key_value = dictionary_utils.get_element(past_folder, key)
                         if key_value is not None:
                             change_folder[key] = key_value
-
-def _add_comment(comment, dictionary):
-    """
-    Add a comment to the specified dictionary
-    :param comment: the comment text
-    :param dictionary: the dictionary to be appended
-    """
-    # make comment key unique, key will not appear in output
-    # comment_key = COMMENT_MATCH + comment
-    # dictionary[comment_key] = comment
-    pass

core/src/main/python/wlsdeploy/yaml/yaml_translator.py

@@ -14,12 +14,13 @@
 import java.lang.Long as JLong
 import java.lang.String as JString
 import java.util.ArrayList as JArrayList
-import java.util.LinkedHashMap as JLinkedHashMap
 
 import oracle.weblogic.deploy.util.FileUtils as JFileUtils
 import oracle.weblogic.deploy.yaml.YamlStreamTranslator as JYamlStreamTranslator
 import oracle.weblogic.deploy.yaml.YamlTranslator as JYamlTranslator
+from oracle.weblogic.deploy.util import OrderedMap
 from oracle.weblogic.deploy.util import PyRealBoolean
+from oracle.weblogic.deploy.util import PyOrderedDict
 
 from wlsdeploy.exception import exception_helper
 from wlsdeploy.logging.platform_logger import PlatformLogger
@@ -122,7 +123,10 @@ def convert_to_java(self):
             raise yaml_ex
 
     def convert_dict_to_java_map(self, dictionary):
-        result = JLinkedHashMap()
+        result = OrderedMap()
+        if isinstance(dictionary, PyOrderedDict):
+            result.setCommentMap(dictionary.getCommentMap())
+
         for key, value in dictionary.iteritems():
             java_key = JString(key)
             if isinstance(value, dict):