From 7837745c226d2a3dd03f2831aec4f8763f8543af Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 14 Jan 2018 20:36:27 +0000 Subject: [PATCH 1/8] * in Lib/uuid.py define ctypes() based definition of util._generate_time_safe based on lib.uuid_create (for when _uuid is not available) * use the character '.' rather than ':' to seperate the MAC ADDR values returned by 'netstat -i' command (AIX specific) * The commands arp and ifconfig do not return a local MAC address --- Lib/test/test_uuid.py | 16 ++++- Lib/uuid.py | 72 ++++++++++++++++--- .../2018-01-14-15-17-10.bpo-28009.bLNd9y.rst | 4 ++ 3 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index f21bd6dfa15235..8cb545d4cda382 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -4,6 +4,7 @@ import contextlib import io import os +import sys import shutil import subprocess @@ -490,10 +491,19 @@ class BaseTestInternals: @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_find_mac(self): - data = ''' + # issue28009 - AIX netstat -ai has specific layout, + # e.g., '.' rather than ':' tsas format character + if not sys.platform.startswith("aix"): + data = ''' fake hwaddr cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab +''' + else: + data = ''' +fake hwaddr +cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 +eth0 Link encap:Ethernet HWaddr 12.34.56.78.90.ab ''' popen = unittest.mock.MagicMock() @@ -523,6 +533,8 @@ def check_node(self, node, requires=None): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): + if sys.platform.startswith("aix"): + self.skipTest('AIX ifconfig does not provide MAC addr') node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @@ -533,6 +545,8 @@ def test_ip_getnode(self): @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): + if sys.platform.startswith("aix"): + self.skipTest('AIX arp does not provide MAC addr') node = self.uuid._arp_getnode() self.check_node(node, 'arp') diff --git a/Lib/uuid.py b/Lib/uuid.py index b7433cb71926ac..dc8fe5d3c3b9e8 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -361,6 +361,11 @@ def _is_universal(mac): return not (mac & (1 << 41)) def _find_mac(command, args, hw_identifiers, get_index): + # issue28009: AIX uses character '.' rather than ':' + if sys.platform.startswith("aix"): + old = b'.' + else: + old = b':' first_local_mac = None try: proc = _popen(command, *args.split()) @@ -373,7 +378,7 @@ def _find_mac(command, args, hw_identifiers, get_index): if words[i] in hw_identifiers: try: word = words[get_index(i)] - mac = int(word.replace(b':', b''), 16) + mac = int(word.replace(old, b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac @@ -391,6 +396,8 @@ def _find_mac(command, args, hw_identifiers, get_index): def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. + if sys.platform.startswith("aix"): + return None keywords = (b'hwaddr', b'ether', b'address:', b'lladdr') for args in ('', '-a', '-av'): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) @@ -401,6 +408,8 @@ def _ifconfig_getnode(): def _ip_getnode(): """Get the hardware address on Unix by running ip.""" # This works on Linux with iproute2. + if sys.platform.startswith("aix"): + return None mac = _find_mac('ip', 'link', [b'link/ether'], lambda i: i+1) if mac: return mac @@ -408,6 +417,8 @@ def _ip_getnode(): def _arp_getnode(): """Get the hardware address on Unix by running arp.""" + if sys.platform.startswith("aix"): + return None import os, socket try: ip_addr = socket.gethostbyname(socket.gethostname()) @@ -434,9 +445,24 @@ def _arp_getnode(): def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" + if sys.platform.startswith("aix"): + return None # This might work on HP-UX. return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0) +def _netstat_getmac_posix(words,i): + """Extract the MAC address from netstat -ia from posix netstat -ia.""" + word = words[i] + if len(word) == 17 and word.count(b':') == 5: + return(int(word.replace(b':', b''), 16)) + +def _netstat_getmac_aix(words,i): + """Extract the MAC address from netstat -ia specific for AIX netstat.""" + word = words[i] + wlen = len(word) + if wlen >= 11 and wlen <=17 and word.count(b'.') == 5: + return int(word.replace(b'.', b''), 16) + def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. @@ -454,12 +480,15 @@ def _netstat_getnode(): for line in proc.stdout: try: words = line.rstrip().split() - word = words[i] - if len(word) == 17 and word.count(b':') == 5: - mac = int(word.replace(b':', b''), 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac + if sys.platform.startswith("aix"): + mac = _netstat_getmac_aix(words,i) + if not mac: + continue + else: + mac = _netstat_getmac_posix(words,i) + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: @@ -528,7 +557,6 @@ def _netbios_getnode(): first_local_mac = first_local_mac or mac return first_local_mac or None - _generate_time_safe = _UuidCreate = None _has_uuid_generate_time_safe = None @@ -577,8 +605,16 @@ def _load_system_functions(): if not sys.platform.startswith('win'): _libnames.append('c') for libname in _libnames: + # issue28009 - (at least) on AIX ctypes.CDLL(None) returns + # the equivalent of libc as that is already dynamically linked + # and anything already dlopen() ed is available + # so, rather than open 'None' several times + # only try to open when something has been found + libnm = ctypes.util.find_library(libname) + if not libnm: + continue try: - lib = ctypes.CDLL(ctypes.util.find_library(libname)) + lib = ctypes.CDLL(libnm) except Exception: # pragma: nocover continue # Try to find the safe variety first. @@ -601,6 +637,22 @@ def _generate_time_safe(): _uuid_generate_time(_buffer) return bytes(_buffer.raw), None break + # issue28009 - (see also #26439, #32399, #32493) + # when find_library("c") does not work AND _uuid is not present + # try to attach to libc using None + if not lib: + try: + lib = ctypes.CDLL(None) + except: + lib = None + if lib: + if hasattr(lib, 'uuid_create'): # pragma: nocover + _uuid_generate_time = lib.uuid_create + def _generate_time_safe(): + _buffer = ctypes.create_string_buffer(16) + _status = ctypes.create_string_buffer(ctypes.sizeof(ctypes.c_ushort)) + _uuid_generate_time(_buffer, _status) + return bytes(_buffer.raw), bytes(_status.raw) # On Windows prior to 2000, UuidCreate gives a UUID containing the # hardware address. On Windows 2000 and later, UuidCreate makes a @@ -670,6 +722,8 @@ def getnode(): if sys.platform == 'win32': getters = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] + elif sys.platform.startswith("aix"): + getters = [_unix_getnode, _netstat_getnode] else: getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] diff --git a/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst b/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst new file mode 100644 index 00000000000000..b3bb91562fdd7e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst @@ -0,0 +1,4 @@ +* AIX uses the character '.' rather than ':' to seperate the MAC ADDR values + returned by 'netstat -i' command. +* The commands arp and ifconfig doe not return a local MAC address +* define util._generate_time_safe based on lib.uuid_create (for when _uuid is not available) From f6d95f3a8d150fc29dcadcdfbc5078d440a4d1af Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 14 Jan 2018 20:47:41 +0000 Subject: [PATCH 2/8] correct typo in News --- .../next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst b/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst index b3bb91562fdd7e..51d11142793cb8 100644 --- a/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst +++ b/Misc/NEWS.d/next/Tests/2018-01-14-15-17-10.bpo-28009.bLNd9y.rst @@ -1,4 +1,5 @@ * AIX uses the character '.' rather than ':' to seperate the MAC ADDR values - returned by 'netstat -i' command. -* The commands arp and ifconfig doe not return a local MAC address -* define util._generate_time_safe based on lib.uuid_create (for when _uuid is not available) + returned by 'netstat -i' command. +* The commands 'arp' and 'ifconfig' do not return a local MAC address +* define uuid._generate_time_safe in uuid.py based on ctypes() + lib.uuid_create (for when _uuid is not available) From 44aa616277737885ed4b2d4facab31a300ad860a Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Sun, 14 Jan 2018 21:03:09 +0000 Subject: [PATCH 3/8] fix whitespace --- Lib/test/test_uuid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 8cb545d4cda382..2f57597d184d98 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -491,8 +491,8 @@ class BaseTestInternals: @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_find_mac(self): - # issue28009 - AIX netstat -ai has specific layout, - # e.g., '.' rather than ':' tsas format character + # issue28009 - AIX netstat -i(a) has specific formatting, + # i.e., '.' rather than ':' as format character if not sys.platform.startswith("aix"): data = ''' fake hwaddr From 64228cd2ee4303fea5ef0cc0472396f5b2df1ac5 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 15 Jan 2018 07:32:12 +0000 Subject: [PATCH 4/8] Ensure that 'lib' is defined before it is used Also, correct comments to explain changes to 'legacy' code --- Lib/uuid.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index dc8fe5d3c3b9e8..b6bb2e9fa898a1 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -601,15 +601,16 @@ def _load_system_functions(): # The uuid_generate_* routines are provided by libuuid on at least # Linux and FreeBSD, and provided by libc on Mac OS X. + # uuid_generate() is provided by libc on AIX + # (and FreeBSD issue 32493) _libnames = ['uuid'] if not sys.platform.startswith('win'): _libnames.append('c') + # attach to a lib and look for uuid_generate* family functions + # rather than open 'None' several times + # only try to dlopen when something has been found + lib = None for libname in _libnames: - # issue28009 - (at least) on AIX ctypes.CDLL(None) returns - # the equivalent of libc as that is already dynamically linked - # and anything already dlopen() ed is available - # so, rather than open 'None' several times - # only try to open when something has been found libnm = ctypes.util.find_library(libname) if not libnm: continue @@ -637,20 +638,24 @@ def _generate_time_safe(): _uuid_generate_time(_buffer) return bytes(_buffer.raw), None break - # issue28009 - (see also #26439, #32399, #32493) - # when find_library("c") does not work AND _uuid is not present - # try to attach to libc using None + # when find_library("c") returns None AND _uuid is not present + # try to attach to libc using ctypes.CDLL(None) + # on AIX (at least) ctypes.CDLL(None) returns + # the equivalent of libc because libc is already dynamically linked + # to the python executable (see also #26439, #32399, #32493) if not lib: try: lib = ctypes.CDLL(None) except: lib = None + # look for uuid_generate() as a way to define both + # _uuid_generate_time and _generate_time_safe if lib: if hasattr(lib, 'uuid_create'): # pragma: nocover _uuid_generate_time = lib.uuid_create def _generate_time_safe(): - _buffer = ctypes.create_string_buffer(16) - _status = ctypes.create_string_buffer(ctypes.sizeof(ctypes.c_ushort)) + _buffer = ctypes.create_string_buffer(16)# uuid + _status = ctypes.create_string_buffer(2) # c_ushort _uuid_generate_time(_buffer, _status) return bytes(_buffer.raw), bytes(_status.raw) From 9d8148ee3d170b7b8fe96bd5ae3ab727dc99ae6a Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 5 Feb 2018 13:57:52 +0000 Subject: [PATCH 5/8] resolve conflicts in uuid.py --- Lib/uuid.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index b6bb2e9fa898a1..2eb10ea37110d4 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -713,6 +713,13 @@ def _random_getnode(): _node = None +_NODE_GETTERS_WIN32 = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] + +_NODE_GETTERS_UNIX = [_unix_getnode, _ifconfig_getnode, _ip_getnode, + _arp_getnode, _lanscan_getnode, _netstat_getnode] + +_NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] + def getnode(): """Get the hardware address as a 48-bit positive integer. @@ -726,12 +733,11 @@ def getnode(): return _node if sys.platform == 'win32': - getters = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] + getters = _NODE_GETTERS_WIN32 elif sys.platform.startswith("aix"): - getters = [_unix_getnode, _netstat_getnode] + getters = _NODE_GETTERS_AIX else: - getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode, - _arp_getnode, _lanscan_getnode, _netstat_getnode] + getters = _NODE_GETTERS_UNIX for getter in getters + [_random_getnode]: try: From e6d1c0ebb8e7761d51eecc461fd9951e95a99b27 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 5 Feb 2018 14:26:56 +0000 Subject: [PATCH 6/8] resolv conflict in uuid.py --- Lib/uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 2eb10ea37110d4..5cafffe42b74b4 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -720,7 +720,7 @@ def _random_getnode(): _NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] -def getnode(): +def getnode(*, getters=None): """Get the hardware address as a 48-bit positive integer. The first time this runs, it may launch a separate program, which could From dd5f39a19955dc8044e92199e50a44eed0709b7b Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 5 Feb 2018 16:01:44 +0000 Subject: [PATCH 7/8] merge with new uuid.py from master --- Lib/uuid.py | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 5cafffe42b74b4..858b4b7f349874 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -363,9 +363,9 @@ def _is_universal(mac): def _find_mac(command, args, hw_identifiers, get_index): # issue28009: AIX uses character '.' rather than ':' if sys.platform.startswith("aix"): - old = b'.' + c_field = b'.' else: - old = b':' + c_field = b':' first_local_mac = None try: proc = _popen(command, *args.split()) @@ -378,7 +378,7 @@ def _find_mac(command, args, hw_identifiers, get_index): if words[i] in hw_identifiers: try: word = words[get_index(i)] - mac = int(word.replace(old, b''), 16) + mac = int(word.replace(c_field, b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac @@ -396,8 +396,6 @@ def _find_mac(command, args, hw_identifiers, get_index): def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. - if sys.platform.startswith("aix"): - return None keywords = (b'hwaddr', b'ether', b'address:', b'lladdr') for args in ('', '-a', '-av'): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) @@ -408,8 +406,6 @@ def _ifconfig_getnode(): def _ip_getnode(): """Get the hardware address on Unix by running ip.""" # This works on Linux with iproute2. - if sys.platform.startswith("aix"): - return None mac = _find_mac('ip', 'link', [b'link/ether'], lambda i: i+1) if mac: return mac @@ -417,8 +413,6 @@ def _ip_getnode(): def _arp_getnode(): """Get the hardware address on Unix by running arp.""" - if sys.platform.startswith("aix"): - return None import os, socket try: ip_addr = socket.gethostbyname(socket.gethostname()) @@ -445,8 +439,6 @@ def _arp_getnode(): def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" - if sys.platform.startswith("aix"): - return None # This might work on HP-UX. return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0) @@ -455,6 +447,8 @@ def _netstat_getmac_posix(words,i): word = words[i] if len(word) == 17 and word.count(b':') == 5: return(int(word.replace(b':', b''), 16)) + else: + return None def _netstat_getmac_aix(words,i): """Extract the MAC address from netstat -ia specific for AIX netstat.""" @@ -462,6 +456,8 @@ def _netstat_getmac_aix(words,i): wlen = len(word) if wlen >= 11 and wlen <=17 and word.count(b'.') == 5: return int(word.replace(b'.', b''), 16) + else: + return None def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" @@ -482,10 +478,10 @@ def _netstat_getnode(): words = line.rstrip().split() if sys.platform.startswith("aix"): mac = _netstat_getmac_aix(words,i) - if not mac: - continue else: mac = _netstat_getmac_posix(words,i) + if not mac: + continue if _is_universal(mac): return mac first_local_mac = first_local_mac or mac @@ -557,6 +553,7 @@ def _netbios_getnode(): first_local_mac = first_local_mac or mac return first_local_mac or None + _generate_time_safe = _UuidCreate = None _has_uuid_generate_time_safe = None @@ -601,14 +598,11 @@ def _load_system_functions(): # The uuid_generate_* routines are provided by libuuid on at least # Linux and FreeBSD, and provided by libc on Mac OS X. - # uuid_generate() is provided by libc on AIX - # (and FreeBSD issue 32493) + # uuid_create() is used by other POSIX platforms (e.g., AIX, FreeBSD) _libnames = ['uuid'] if not sys.platform.startswith('win'): _libnames.append('c') - # attach to a lib and look for uuid_generate* family functions - # rather than open 'None' several times - # only try to dlopen when something has been found + # also need to look for uuid_create() but only look if something was located lib = None for libname in _libnames: libnm = ctypes.util.find_library(libname) @@ -638,18 +632,14 @@ def _generate_time_safe(): _uuid_generate_time(_buffer) return bytes(_buffer.raw), None break - # when find_library("c") returns None AND _uuid is not present - # try to attach to libc using ctypes.CDLL(None) - # on AIX (at least) ctypes.CDLL(None) returns - # the equivalent of libc because libc is already dynamically linked - # to the python executable (see also #26439, #32399, #32493) + # if _uuid_generate_time has not been found (Linux) then we try libc + # as libc is already dynamically linked (or found above) verify a valid + # lib value, then look for uuid_generate() if not lib: try: lib = ctypes.CDLL(None) except: lib = None - # look for uuid_generate() as a way to define both - # _uuid_generate_time and _generate_time_safe if lib: if hasattr(lib, 'uuid_create'): # pragma: nocover _uuid_generate_time = lib.uuid_create @@ -718,7 +708,7 @@ def _random_getnode(): _NODE_GETTERS_UNIX = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] -_NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] +_NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] def getnode(*, getters=None): """Get the hardware address as a 48-bit positive integer. @@ -744,9 +734,9 @@ def getnode(*, getters=None): _node = getter() except: continue - if _node is not None: + if (_node is not None) and (0 <= _node < (1 << 48)): return _node - assert False, '_random_getnode() returned None' + assert False, '_random_getnode() returned invalid value: {}'.format(_node) _last_timestamp = None From 7b25dd35c3cee334a68eff7ee02512b7d1d3b5a9 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Mon, 5 Feb 2018 16:09:13 +0000 Subject: [PATCH 8/8] uuid.py does not merge my additions, will try without the added efficiency --- Lib/uuid.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 858b4b7f349874..2fdd59c3802ac7 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -708,8 +708,6 @@ def _random_getnode(): _NODE_GETTERS_UNIX = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] -_NODE_GETTERS_AIX = [_unix_getnode, _netstat_getnode] - def getnode(*, getters=None): """Get the hardware address as a 48-bit positive integer. @@ -724,8 +722,6 @@ def getnode(*, getters=None): if sys.platform == 'win32': getters = _NODE_GETTERS_WIN32 - elif sys.platform.startswith("aix"): - getters = _NODE_GETTERS_AIX else: getters = _NODE_GETTERS_UNIX