Allow late binding of file and properties for command line validation (#443)

* introduced lax validation for command line validation tool to allow late binding of @@file@@ and @@prop@@ values in the model.

Author: jshum2479

Jenkinsfile

@@ -37,4 +37,4 @@ pipeline {
             }
         }
     }
-}
+}

core/src/main/python/create.py

@@ -106,7 +106,7 @@ def __process_args(args):
     __process_rcu_args(optional_arg_map, domain_type, domain_typedef)
     __process_encryption_args(optional_arg_map)
     __process_opss_args(optional_arg_map)
-    
+
     combined_arg_map = optional_arg_map.copy()
     combined_arg_map.update(required_arg_map)
     model_context = ModelContext(_program_name, combined_arg_map)

core/src/main/python/validate.py

@@ -46,7 +46,8 @@
     CommandLineArgUtil.TARGET_MODE_SWITCH,
     CommandLineArgUtil.ATTRIBUTES_ONLY_SWITCH,
     CommandLineArgUtil.FOLDERS_ONLY_SWITCH,
-    CommandLineArgUtil.RECURSIVE_SWITCH
+    CommandLineArgUtil.RECURSIVE_SWITCH,
+    CommandLineArgUtil.VALIDATION_METHOD
 ]
 
 
@@ -247,6 +248,7 @@ def main(args):
                     cla_helper.clean_up_temp_files()
                     sys.exit(CommandLineArgUtil.PROG_WARNING_EXIT_CODE)
 
+
         except ValidateException, ve:
             __logger.severe('WLSDPLY-20000', _program_name, ve.getLocalizedMessage(), error=ve,
                             class_name=_class_name, method_name=_method_name)

core/src/main/python/wlsdeploy/aliases/model_constants.py

@@ -111,6 +111,7 @@
 FOREIGN_SERVER = 'ForeignServer'
 GROUP = 'Group'
 GROUP_PARAMS = 'GroupParams'
+GLOBAL_VARIABLE_SUBSTITUTION = 'VariableSubstitution'
 HARVESTED_TYPE = 'HarvestedType'
 HARVESTER = 'Harvester'
 HEAP_DUMP_ACTION = 'HeapDumpAction'

core/src/main/python/wlsdeploy/tool/validate/validation_results.py

@@ -13,6 +13,7 @@
 
 from wlsdeploy.util import model
 from wlsdeploy.tool.validate import validation_utils
+from wlsdeploy.aliases import model_constants
 
 
 class ValidationResults(object):
@@ -27,7 +28,8 @@ def __init__(self):
             '%s Section' % model.get_model_domain_info_key(): None,
             '%s Section' % model.get_model_topology_key(): None,
             '%s Section' % model.get_model_deployments_key(): None,
-            '%s Section' % model.get_model_resources_key(): None
+            '%s Section' % model.get_model_resources_key(): None,
+            '%s Section' % model_constants.GLOBAL_VARIABLE_SUBSTITUTION: None
         }
 
     def __str__(self):

core/src/main/python/wlsdeploy/tool/validate/validator.py

@@ -44,6 +44,8 @@
 _TOPOLOGY_VALIDATION_AREA = validation_utils.format_message('WLSDPLY-05001', model_constants.TOPOLOGY)
 _RESOURCES_VALIDATION_AREA = validation_utils.format_message('WLSDPLY-05001', model_constants.RESOURCES)
 _APP_DEPLOYMENTS_VALIDATION_AREA = validation_utils.format_message('WLSDPLY-05001', model_constants.APP_DEPLOYMENTS)
+_GLOBAL_LEVEL_VARAIBLE_SUBSTITUTE = validation_utils.format_message('WLSDPLY-05001',
+                                                                    model_constants.GLOBAL_VARIABLE_SUBSTITUTION)
 
 
 class Validator(object):
@@ -239,8 +241,11 @@ def __validate_model_file(self, model_dict, variables_file_name, archive_file_na
             if variables_file_name is not None:
                 self._logger.info('WLSDPLY-05004', variables_file_name, class_name=_class_name, method_name=_method_name)
                 self._variable_properties = variables.load_variables(variables_file_name)
+            validation_result = ValidationResult(_GLOBAL_LEVEL_VARAIBLE_SUBSTITUTE)
+            validation_result = variables.substitute(model_dict, self._variable_properties, self._model_context,
+                                                     validation_result)
 
-            variables.substitute(model_dict, self._variable_properties, self._model_context)
+            self._validation_results.set_validation_result(validation_result)
         except VariableException, ve:
             ex = exception_helper.create_validate_exception('WLSDPLY-20004', 'validateModel',
                                                             ve.getLocalizedMessage(), error=ve)

core/src/main/python/wlsdeploy/util/cla_utils.py

@@ -64,6 +64,7 @@ class CommandLineArgUtil(object):
     ATTRIBUTES_ONLY_SWITCH     = '-attributes_only'
     FOLDERS_ONLY_SWITCH        = '-folders_only'
     RECURSIVE_SWITCH           = '-recursive'
+    VALIDATION_METHOD          = '-method'
     # overrides for the variable injector
     VARIABLE_INJECTOR_FILE_SWITCH   = '-variable_injector_file'
     VARIABLE_KEYWORDS_FILE_SWITCH   = '-variable_keywords_file'
@@ -280,6 +281,15 @@ def process_args(self, args, for_domain_create=False):
                     ex = self._get_out_of_args_exception(key)
                     self._logger.throwing(ex, class_name=self._class_name, method_name=method_name)
                     raise ex
+            elif self.is_validate_method_key(key):
+                idx += 1
+                if idx < args_len:
+                    context = self._validate_validate_method_arg(args[idx])
+                    self._add_arg(key, context)
+                else:
+                    ex = self._get_out_of_args_exception(key)
+                    self._logger.throwing(ex, class_name=self._class_name, method_name=method_name)
+                    raise ex
             elif self.is_variable_file_key(key):
                 idx += 1
                 if idx < args_len:
@@ -757,6 +767,24 @@ def _validate_previous_model_file_arg(self, value):
             raise ex
         return model.getAbsolutePath()
 
+    def is_validate_method_key(self, key):
+        return self.VALIDATION_METHOD == key
+
+    def _validate_validate_method_arg(self, value):
+        method_name = '_validate_validate_method_arg'
+
+        if value is None or len(value) == 0:
+            ex = exception_helper.create_cla_exception('WLSDPLY-20029')
+            ex.setExitCode(self.ARG_VALIDATION_ERROR_EXIT_CODE)
+            self._logger.throwing(ex, class_name=self._class_name, method_name=method_name)
+            raise ex
+        elif value.lower() != 'strict' and value.lower() != 'lax':
+            ex = exception_helper.create_cla_exception('WLSDPLY-20030', value, "strict, or lax")
+            ex.setExitCode(self.ARG_VALIDATION_ERROR_EXIT_CODE)
+            self._logger.throwing(ex, class_name=self._class_name, method_name=method_name)
+            raise ex
+        return value
+
     def get_print_usage_key(self):
         return self.PRINT_USAGE_SWITCH
 

core/src/main/python/wlsdeploy/util/model_context.py

@@ -63,6 +63,7 @@ def __init__(self, program_name, arg_map):
         self._folders_only = False
         self._opss_wallet_passphrase = None
         self._opss_wallet = None
+        self._validation_method = None
 
         if CommandLineArgUtil.ORACLE_HOME_SWITCH in arg_map:
             self._oracle_home = arg_map[CommandLineArgUtil.ORACLE_HOME_SWITCH]
@@ -153,6 +154,9 @@ def __init__(self, program_name, arg_map):
         if CommandLineArgUtil.OPSS_WALLET_SWITCH in arg_map:
             self._opss_wallet = arg_map[CommandLineArgUtil.OPSS_WALLET_SWITCH]
 
+        if CommandLineArgUtil.VALIDATION_METHOD in arg_map:
+            self._validation_method = arg_map[CommandLineArgUtil.VALIDATION_METHOD]
+
         if CommandLineArgUtil.TARGET_VERSION_SWITCH in arg_map:
             self._wl_version = arg_map[CommandLineArgUtil.TARGET_VERSION_SWITCH]
 
@@ -287,6 +291,14 @@ def get_opss_wallet_passphrase(self):
         """
         return self._opss_wallet_passphrase
 
+    def get_validation_method(self):
+        """
+        Get the validation method.
+        :return: the validation method
+        """
+        if self._validation_method is None:
+            self._validation_method = 'strict'
+        return self._validation_method
 
     def get_archive_file(self):
         """

core/src/main/python/wlsdeploy/util/variables.py

@@ -134,17 +134,18 @@ def get_variable_names(text):
     return names
 
 
-def substitute(dictionary, variables, model_context):
+def substitute(dictionary, variables, model_context, validation_result=None):
     """
     Substitute fields in the specified dictionary with variable values.
     :param dictionary: the dictionary in which to substitute variables
     :param variables: a dictionary of variables for substitution
     :param model_context: used to resolve variables in file paths
     """
-    _process_node(dictionary, variables, model_context)
+    _process_node(dictionary, variables, model_context, validation_result)
+    return validation_result
 
 
-def _process_node(nodes, variables, model_context):
+def _process_node(nodes, variables, model_context, validation_result):
     """
     Process variables in the node.
     :param nodes: the dictionary to process
@@ -160,18 +161,18 @@ def _process_node(nodes, variables, model_context):
         value = nodes[key]
 
         # if the key changes with substitution, remove old key and map value to new key
-        new_key = _substitute(key, variables, model_context)
+        new_key = _substitute(key, variables, model_context, validation_result)
         if new_key is not key:
             nodes.pop(key)
             nodes[new_key] = value
 
         if isinstance(value, dict):
-            _process_node(value, variables, model_context)
+            _process_node(value, variables, model_context, validation_result)
         elif type(value) is str:
-            nodes[key] = _substitute(value, variables, model_context)
+            nodes[key] = _substitute(value, variables, model_context, validation_result)
 
 
-def _substitute(text, variables, model_context):
+def _substitute(text, variables, model_context, validation_result):
     """
     Substitute the variable placeholders with the variable value.
     :param text: the text to process for variable placeholders
@@ -204,17 +205,25 @@ def _substitute(text, variables, model_context):
                 key = token[7:-2]
                 # for @@PROP:key@@ variables, throw an exception if key is not found.
                 if key not in variables:
-                    ex = exception_helper.create_variable_exception('WLSDPLY-01732', key)
-                    _logger.throwing(ex, class_name=_class_name, method_name=method_name)
-                    raise ex
+                    if model_context.get_validation_method() == 'strict':
+                        if validation_result:
+                            validation_result.add_error('WLSDPLY-01732', key)
+                        ex = exception_helper.create_variable_exception('WLSDPLY-01732', key)
+                        _logger.throwing(ex, class_name=_class_name, method_name=method_name)
+                        raise ex
+                    else:
+                        if validation_result:
+                            validation_result.add_info('WLSDPLY-01732', key)
+                            continue
+                            
                 value = variables[key]
                 text = text.replace(token, value)
 
         tokens = _file_variable_pattern.findall(text)
         if tokens:
             for token in tokens:
                 path = token[7:-2]
-                value = _read_value_from_file(path)
+                value = _read_value_from_file(path, model_context, validation_result)
                 text = text.replace(token, value)
 
         # special case for @@FILE:@@ORACLE_HOME@@/dir/name.txt@@
@@ -223,13 +232,13 @@ def _substitute(text, variables, model_context):
             for token in tokens:
                 path = token[7:-2]
                 path = model_context.replace_token_string(path)
-                value = _read_value_from_file(path)
+                value = _read_value_from_file(path, model_context, validation_result)
                 text = text.replace(token, value)
 
     return text
 
 
-def _read_value_from_file(file_path):
+def _read_value_from_file(file_path, model_context, validation_result):
     """
     Read a single text value from the first line in the specified file.
     :param file_path: the file from which to read the value
@@ -243,9 +252,18 @@ def _read_value_from_file(file_path):
         line = file_reader.readLine()
         file_reader.close()
     except IOException, e:
-        ex = exception_helper.create_variable_exception('WLSDPLY-01733', file_path, e.getLocalizedMessage(), error=e)
-        _logger.throwing(ex, class_name=_class_name, method_name=method_name)
-        raise ex
+        if model_context.get_validation_method() == 'strict':
+            if validation_result:
+                validation_result.add_error('WLSDPLY-01733', file_path, e.getLocalizedMessage())
+            ex = exception_helper.create_variable_exception('WLSDPLY-01733', file_path, e.getLocalizedMessage(), error=e)
+            _logger.throwing(ex, class_name=_class_name, method_name=method_name)
+            raise ex
+        else:
+            if validation_result:
+                validation_result.add_info('WLSDPLY-01733', file_path, e.getLocalizedMessage())
+            _logger.info('WLSDPLY-01733', file_path, e.getLocalizedMessage(), error=e,
+                            class_name=_class_name, method_name=method_name)
+            line = ''
 
     if line is None:
         ex = exception_helper.create_variable_exception('WLSDPLY-01734', file_path)

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -1446,6 +1446,8 @@ WLSDPLY-20025=For {0}, specify the {1} or {2} argument, but not both
 WLSDPLY-20026={0} failed to find a model file in archive {1}, and {2} argument not specified
 WLSDPLY-20027=Enter the OPSS wallet passphrase
 WLSDPLY-20028=Failed to read the OPSS wallet passphrase input from the user: {0}
+WLSDPLY-20029=Specified validation method to use was empty or null
+WLSDPLY-20030=Specified validation method {0} is invalid, must be one of: {1}
 
 # Common messages used for tool exit and clean-up
 WLSDPLY-21000={0} Messages:

installer/src/main/bin/validateModel.cmd

@@ -15,20 +15,61 @@
 @rem This script uses the following command-line arguments directly, the rest
 @rem of the arguments are passed down to the underlying python program:
 @rem
-@rem     - -oracle_home        The directory of the existing Oracle Home to use.
-@rem                           This directory must exist and it is the caller^'s
-@rem                           responsibility to verify that it does. This
-@rem                           argument is required.
+@rem     - -oracle_home       The directory of the existing Oracle Home to use.
+@rem                          This directory must exist and it is the caller^'s
+@rem                          responsibility to verify that it does. This
+@rem                          argument is required.
 @rem
-@rem     - -domain_type        The type of domain to create. This argument is
-@rem                           is optional.  If not specified, it defaults to WLS.
+@rem     - -wlst_path         The path to the Oracle Home product directory under
+@rem                          which to find the wlst.cmd script.  This is only
+@rem                          needed for pre-12.2.1 upper stack products like SOA.
 @rem
-@rem     - -wlst_path          The path to the Oracle Home product directory under
-@rem                           which to find the wlst.cmd script.  This is only
-@rem                           needed for pre-12.2.1 upper stack products like SOA.
+@rem                          For example, for SOA 12.1.3, -wlst_path should be
+@rem                          specified as %ORACLE_HOME%\soa
 @rem
-@rem                           For example, for SOA 12.1.3, -wlst_path should be
-@rem                           specified as %ORACLE_HOME%\soa
+@rem     - -domain_type       The type of domain (e.g., WLS, JRF).
+@rem                          Used to locate wlst.cmd if -wlst_path not specified
+@rem
+@rem The following arguments are passed down to the underlying python program:
+@rem
+@rem     - -print_usage       Specify the context for printing out the model structure.
+@rem                          By default, the specified folder attributes and subfolder
+@rem                          names are printed.  Use one of the optional control
+@rem                          switches to customize the behavior.  Note that the
+@rem                          control switches are mutually exclusive.
+@rem
+@rem     - -model_file        The location of the model file to use if not using
+@rem                          the -print_usage functionality.  This can also be specified as a
+@rem                          comma-separated list of model locations, where each successive model layers
+@rem                          on top of the previous ones.  If not specified, the tool will look for the
+@rem                          model in the archive.  If the model is not found, validation will only
+@rem                          validate the artifacts provided.
+@rem
+@rem     - -variable_file     The location of the property file containing
+@rem                          the variable values for all variables used in
+@rem                          the model if not using the -print_usage functionality.
+@rem                          If the variable file is not provided, validation will
+@rem                          only validate the artifacts provided.
+@rem
+@rem     - -archive_file      The path to the archive file to use if not using the
+@rem                          -print_usage functionality.  If the archive file is
+@rem                          not provided, validation will only validate the
+@rem                          artifacts provided.
+@rem
+@rem     - -target_version    The target version of WebLogic Server the tool
+@rem                          should use to validate the model content.  This
+@rem                          version number can be different than the version
+@rem                          being used to run the tool.  If not specified, the
+@rem                          tool will validate against the version being used
+@rem                          to run the tool.
+@rem
+@rem     - -target_mode       The target WLST mode that the tool should use to
+@rem                          validate the model content.  The only valid values
+@rem                          are online or offline.  If not specified, the tool
+@rem                          defaults to WLST offline mode.
+@rem
+@rem     - -method            The validation method to apply. Options: lax, strict.
+@rem                          The lax method will skip validation of external model references like @@FILE@@
 @rem
 @rem This script uses the following variables:
 @rem

installer/src/main/bin/validateModel.sh

@@ -58,11 +58,12 @@ usage() {
   echo "          [-target_mode <target_mode>]"
   echo "          [-domain_type <domain_type>]"
   echo "          [-wlst_path <wlst_path>]"
+  echo "          [-method <method>]"
   echo ""
   echo "    where:"
   echo "        oracle_home     - the existing Oracle Home directory for the domain"
   echo ""
-  echo "        context         - specify the context for printing out the model structure."
+  echo "        print_usage     - specify the context for printing out the model structure."
   echo "                          By default, the specified folder attributes and subfolder"
   echo "                          names are printed.  Use one of the optional control"
   echo "                          switches to customize the behavior.  Note that the"
@@ -104,6 +105,9 @@ usage() {
   echo "        wlst_path       - the Oracle Home subdirectory of the wlst.cmd"
   echo "                          script to use (e.g., <ORACLE_HOME>/soa)"
   echo ""
+  echo "        method          - the validation method to apply. Options: lax, strict. "
+  echo "                          The lax method will skip validation of external model references like @@FILE@@"
+  echo ""
 }
 
 umask 27