Updates to supported versions, handing Template deprecation in NiFi-2.x, and Windows development support

README.rst

@@ -96,9 +96,9 @@ Background and Documentation
 NiFi Version Support
 --------------------
 
-| Currently we are testing against NiFi versions 1.9.2 - 1.27.0, and NiFi-Registry versions 0.3.0 - 1.27.0.
+| Currently we are testing against NiFi versions 1.9.2 - 1.28.1, and NiFi-Registry versions 0.3.0 - 1.28.1.
 
-| We have also tested against the latest 2.0.0-M4 release candidates using the 1.x SDK and have found that basic functionality works as expected.
+| We have also tested against the latest NiFi-2.2.0 release using the 1.x SDK and have found that basic functionality works as expected.
 | Apache NiFi offers no compatibility guarantees between major versions, and while many functions may work the same, you should test carefully for your specific use case.
 | In future we will create a specific branch of NiPyAPI for NiFi 2.x SDKs, and maintain separate NiFi 1.x and NiFi 2.x clients.
 
@@ -107,9 +107,9 @@ NiFi Version Support
 Python Support
 --------------
 
-| Python 2.7 or 3.7-12 supported, though other versions may work. 
+| Python 2.7 or 3.9-12 supported, though other versions may work. 
 | We will shortly stop supporting Python2.
-| OSX M1 chips have various issues with Requests and Certificates.
+| OSX M1 chips have had various issues with Requests and Certificates.
 
-| Tested on AL2023, developed on OSX 14.2 - Windows testing not attempted.
+| Tested on AL2023, developed on OSX 14+ and Windows 10 with Docker Desktop.
 | Outside of the standard Python modules, we make use of lxml, DeepDiff, ruamel.yaml and xmltodict in processing, and Docker for demo/tests.

nipyapi/canvas.py

@@ -149,6 +149,7 @@ def get_process_group(identifier, identifier_type='name', greedy=True):
     return out
 
 
+# pylint: disable=R1737
 def list_all_process_groups(pg_id='root'):
     """
     Returns a flattened list of all Process Groups on the canvas.
@@ -403,7 +404,7 @@ def delete_process_group(process_group, force=False, refresh=True):
                         removed_controllers_id.append(x.component.id)
 
             # Templates are not supported in NiFi 2.x
-            if nipyapi.utils.check_version('2', service='nifi') < 0:
+            if nipyapi.utils.check_version('2', service='nifi') == 1:
                 for template in nipyapi.templates.list_all_templates(native=False):
                     if target.id == template.template.group_id:
                         nipyapi.templates.delete_template(template.id)

nipyapi/security.py

@@ -721,7 +721,7 @@ def create_access_policy(resource, action, r_id=None, service="nifi"):
         )
 
 
-# pylint: disable=R0913
+# pylint: disable=R0913, R0917
 def set_service_ssl_context(
     service="nifi",
     ca_file=None,

nipyapi/templates.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-For Managing NiFi Templates
+For Managing NiFi Templates in NiFi 1.x
 """
 
 from __future__ import absolute_import
@@ -94,6 +94,7 @@ def deploy_template(pg_id, template_id, loc_x=0.0, loc_y=0.0):
             template
 
     """
+    nipyapi.utils.validate_templates_version_support()
     with nipyapi.utils.rest_exceptions():
         return nipyapi.nifi.ProcessGroupsApi().instantiate_template(
             id=pg_id,
@@ -146,6 +147,7 @@ def create_template(pg_id, name, desc=''):
         (TemplateEntity): The newly created Template
 
     """
+    nipyapi.utils.validate_templates_version_support()
     snippet = create_pg_snippet(pg_id)
     with nipyapi.utils.rest_exceptions():
         new_template = nipyapi.nifi.CreateTemplateRequestEntity(
@@ -169,6 +171,7 @@ def delete_template(t_id):
     Returns:
         The updated Template object
     """
+    nipyapi.utils.validate_templates_version_support()
     with nipyapi.utils.rest_exceptions():
         return nipyapi.nifi.TemplatesApi().remove_template(id=t_id)
 
@@ -186,6 +189,7 @@ def upload_template(pg_id, template_file):
         (TemplateEntity): The new Template object
 
     """
+    nipyapi.utils.validate_templates_version_support()
     with nipyapi.utils.rest_exceptions():
         this_pg = nipyapi.canvas.get_process_group(pg_id, 'id')
         assert isinstance(this_pg, nipyapi.nifi.ProcessGroupEntity)
@@ -238,6 +242,7 @@ def export_template(t_id, output='string', file_path=None):
             that this may not be utf-8 encoded.
 
     """
+    nipyapi.utils.validate_templates_version_support()
     assert output in ['string', 'file']
     assert file_path is None or isinstance(file_path, six.string_types)
     template = nipyapi.templates.get_template(t_id, 'id')
@@ -261,6 +266,7 @@ def list_all_templates(native=True):
     Returns:
         (list[TemplateEntity]): A list of TemplateEntity's
     """
+    nipyapi.utils.validate_templates_version_support()
     with nipyapi.utils.rest_exceptions():
         templates = nipyapi.nifi.FlowApi().get_templates()
     if not native:

nipyapi/utils.py

@@ -349,7 +349,7 @@ def set_endpoint(endpoint_url, ssl=False, login=False,
     return True
 
 
-# pylint: disable=R0913,R0902
+# pylint: disable=R0913,R0902,R0917
 class DockerContainer():
     """
     Helper class for Docker container automation without using Ansible
@@ -478,6 +478,10 @@ def start_docker_containers(docker_containers, network_name='demo'):
         ))
 
 
+class VersionError(Exception):
+    """Error raised when a feature is not supported in the current version"""
+
+
 def check_version(base, comparator=None, service='nifi',
                   default_version='0.2.0'):
     """
@@ -498,6 +502,8 @@ def check_version(base, comparator=None, service='nifi',
 
     Returns (int): -1/0/1 if base is lower/equal/newer than comparator
 
+    Raises:
+        VersionError: When a feature is not supported in the current version
     """
 
     def strip_version_string(version_string):
@@ -571,6 +577,25 @@ def validate_parameters_versioning_support(verify_nifi=True,
                     "Version Control")
 
 
+def validate_templates_version_support():
+    """
+    Validate that the current version of NiFi supports Templates API
+    """
+    enforce_max_ver('2', service='nifi', error_message="Templates are deprecated in NiFi 2.x")
+
+
+def enforce_max_ver(max_version, bool_response=False, service='nifi', error_message=None):
+    """
+    Raises an error if target NiFi environment is at or above the max version
+    """
+    if check_version(max_version, service=service) == -1:
+        if not bool_response:
+            raise VersionError(error_message or "This function is not available "
+                               "in NiFi {} or above".format(max_version))
+        return True
+    return False
+
+
 def enforce_min_ver(min_version, bool_response=False, service='nifi'):
     """
     Raises an error if target NiFi environment is not minimum version
@@ -585,7 +610,7 @@ def enforce_min_ver(min_version, bool_response=False, service='nifi'):
     """
     if check_version(min_version, service=service) == 1:
         if not bool_response:
-            raise NotImplementedError(
+            raise VersionError(
                 "This function is not available "
                 "before NiFi version " + str(min_version))
         return True

nipyapi/versioning.py

@@ -42,7 +42,7 @@ def create_registry_client(name, uri, description, reg_type=None):
     assert isinstance(uri, six.string_types) and uri is not False
     assert isinstance(name, six.string_types) and name is not False
     assert isinstance(description, six.string_types)
-    if nipyapi.utils.check_version('2', service='nifi') > 0:
+    if nipyapi.utils.check_version('2', service='nifi') == 1:
         component = {
             'uri': uri,
             'name': name,
@@ -231,7 +231,7 @@ def get_flow_in_bucket(bucket_id, identifier, identifier_type='name',
         obj, identifier, identifier_type, greedy=greedy)
 
 
-# pylint: disable=R0913
+# pylint: disable=R0913,R0917
 def save_flow_ver(process_group, registry_client, bucket, flow_name=None,
                   flow_id=None, comment='', desc='', refresh=True,
                   force=False):
@@ -710,7 +710,7 @@ def import_flow_version(bucket_id, encoded_flow=None, file_path=None,
     )
 
 
-# pylint: disable=R0913
+# pylint: disable=R0913, R0917
 def deploy_flow_version(parent_id, location, bucket_id, flow_id, reg_client_id,
                         version=None):
     """

resources/docker/tox-full/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       - SINGLE_USER_CREDENTIALS_PASSWORD=supersecret1!
       - NIFI_WEB_HTTPS_PORT=10127
   nifi:
-    image: apache/nifi:2.0.0-M4
+    image: apache/nifi:2.2.0
     container_name: nifi
     hostname: nifi
     ports:
@@ -41,7 +41,7 @@ services:
     environment:
       - NIFI_REGISTRY_WEB_HTTP_PORT=18127
   registry:
-    image: apache/nifi-registry:2.0.0-M4
+    image: apache/nifi-registry:2.2.0
     container_name: registry
     hostname: registry
     ports:

setup.py

@@ -58,7 +58,10 @@
         "Programming Language :: Python :: 2",
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Topic :: Software Development :: User Interfaces'
     ],
     test_suite='tests'

tests/conftest.py

@@ -235,11 +235,15 @@ def session_setup(request):
 
 
 def remove_test_templates():
-    all_templates = nipyapi.templates.list_all_templates(native=False)
-    if all_templates is not None:
-        for this_template in all_templates:
-            if test_basename in this_template.template.name:
-                nipyapi.templates.delete_template(this_template.id)
+    if nipyapi.utils.enforce_max_ver('2', True):
+        pass
+        # Templates are not supported in NiFi 2
+    else:
+        all_templates = nipyapi.templates.list_all_templates(native=False)
+        if all_templates is not None:
+            for this_template in all_templates:
+                if test_basename in this_template.template.name:
+                    nipyapi.templates.delete_template(this_template.id)
 
 
 def remove_test_pgs():
@@ -292,6 +296,9 @@ def final_cleanup():
             cleanup_nifi()
         elif 'nifi-registry-api' in url:
             cleanup_reg()
+        if test_security and 'https' in url:
+            remove_test_service_user_groups('nifi')
+            remove_test_service_users('nifi')
 
 
 def remove_test_service_users(service='both'):
@@ -333,7 +340,7 @@ def cleanup_nifi():
     log.info("Bulk cleanup called on host %s",
              nipyapi.config.nifi_config.host)
     # Check if NiFi version is 2 or newer
-    if nipyapi.utils.check_version('2', service='nifi') < 0:
+    if nipyapi.utils.check_version('2', service='nifi') == 1:  # We're on an older version
         remove_test_templates()
     remove_test_pgs()
     remove_test_connections()
@@ -343,10 +350,6 @@ def cleanup_nifi():
     remove_test_funnels()
     remove_test_rpgs()
     remove_test_parameter_contexts()
-    if test_security and 'https' in nipyapi.nifi.configuration.host:
-        remove_test_service_user_groups('nifi')
-        remove_test_service_users('nifi')
-
 
 def remove_test_rpgs():
     _ = [
@@ -627,7 +630,7 @@ def __call__(self, parent_pg=None, kind=None):
                 )
             else:
                 target_pg = parent_pg
-            kind = kind if kind else 'DistributedMapCacheClientService'
+            kind = kind if kind else 'CSVReader'
             cont_type = [
                 x for x in nipyapi.canvas.list_all_controller_types()
                 if kind in x.type

tests/test_canvas.py

@@ -545,16 +545,14 @@ def test_get_controller(regress_nifi, fix_pg, fix_cont):
     assert isinstance(r1, nifi.ControllerServiceEntity)
     r2 = canvas.get_controller(f_c1.component.name)
     assert r2.component.name == f_c1.component.name
-    _ = fix_cont(parent_pg=f_pg, kind='DistributedMapCacheServer')
-    r3 = canvas.get_controller('DistributedMapCache')
+    _ = fix_cont(parent_pg=f_pg, kind='CSVReader')
+    r3 = canvas.get_controller('CSVReader')
     assert len(r3) == 2
 
 
 def test_schedule_controller(regress_nifi, fix_pg, fix_cont):
     f_pg = fix_pg.generate()
     f_c1 = fix_cont(parent_pg=f_pg)
-    f_c1 = canvas.update_controller(
-        f_c1, nifi.ControllerServiceDTO(properties={'Server Hostname': 'Bob'}))
     with pytest.raises(AssertionError):
         _ = canvas.schedule_controller('pie', False)
     with pytest.raises(AssertionError):
@@ -571,8 +569,6 @@ def test_delete_controller(regress_nifi, fix_pg, fix_cont):
     r1 = canvas.delete_controller(f_c1)
     assert r1.revision is None
     f_c2 = fix_cont(parent_pg=f_pg)
-    f_c2 = canvas.update_controller(
-        f_c2, nifi.ControllerServiceDTO(properties={'Server Hostname': 'Bob'}))
     f_c2 = canvas.schedule_controller(f_c2, True)
     with pytest.raises(AssertionError):
         _ = canvas.delete_controller('pie')