From 7866f7a6d12cb58c34e0bd8fd427f867b749ef9c Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 27 Nov 2024 00:38:20 -0800 Subject: [PATCH 01/58] [UPDATE] back-ported tested improvements from multicast project to this template (- WIP #79 -) --- Makefile | 313 ++++++++++++++++++++++++++++------- pyproject.tomal | 12 ++ setup.cfg | 58 +++---- setup.py | 2 +- tests/check_cc_lines | 166 +++++++++++++++++++ tests/check_spelling | 221 +++++++++++++++++++++++-- tests/context.py | 58 ++++++- tests/fetch_cc-test-reporter | 93 +++++++---- 8 files changed, 781 insertions(+), 142 deletions(-) create mode 100644 pyproject.tomal create mode 100755 tests/check_cc_lines diff --git a/Makefile b/Makefile index cc86c25..8bf77be 100644 --- a/Makefile +++ b/Makefile @@ -17,16 +17,90 @@ # limitations under the License. +ifeq "$(LC_CTYPE)" "" + LC_CTYPE="en_US.UTF-8" +endif + +ifndef SHELL + SHELL:=command -pv bash +endif + +ifeq "$(ERROR_LOG_PATH)" "" + ERROR_LOG_PATH="/dev/null" +endif + +ifeq "$(COMMAND)" "" + COMMAND_CMD!=`command -v xcrun || command which which || command -v which || command -v command` + ifeq "$(COMMAND_CMD)" "*xcrun" + COMMAND_ARGS=--find + endif + ifeq "$(COMMAND_CMD)" "*command" + COMMAND_ARGS=-pv + endif + COMMAND=$(COMMAND_CMD) $(COMMAND_ARGS) +endif + +ifeq "$(MAKE)" "" + # just no cmake please + MAKEFLAGS=$(MAKEFLAGS) -s + MAKE!=`$(COMMAND) make 2>$(ERROR_LOG_PATH) || $(COMMAND) gnumake 2>$(ERROR_LOG_PATH)` +endif + ifeq "$(ECHO)" "" - ECHO=echo + ECHO=printf "%s\n" +endif + +ifdef "$(ACTION)" + SET_FILE_ATTR=$(COMMAND) xattr +endif + +ifdef "$(SET_FILE_ATTR)" + CREATEDBYBUILDSYSTEM=-w com.apple.xcode.CreatedByBuildSystem true + BSMARK=$(SET_FILE_ATTR) $(CREATEDBYBUILDSYSTEM) +else + BSMARK=$(COMMAND) touch -a endif ifeq "$(LINK)" "" LINK=ln -sf endif -ifeq "$(MAKE)" "" - MAKE=make +ifeq "$(PYTHON)" "" + PY_CMD=$(COMMAND) python3 + ifneq "$(PY_CMD)" "" + PY_ARGS=-B + else + PY_CMD=$(COMMAND) python + endif + PYTHON=$(PY_CMD) $(PY_ARGS) + ifeq "$(COVERAGE)" "" + COVERAGE=$(PYTHON) -m coverage + #COV_CORE_SOURCE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/ + COV_CORE_CONFIG = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.coveragerc + COV_CORE_DATAFILE = .coverage + endif + ifeq "$(COVERAGE)" "" + COVERAGE!=$(COMMAND) coverage + endif +else + ifeq "$(COVERAGE)" "" + COVERAGE!=$(COMMAND) coverage + #COV_CORE_SOURCE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/ + COV_CORE_CONFIG = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.coveragerc + COV_CORE_DATAFILE = .coverage + endif +endif + +ifndef PIP_COMMON_FLAGS + # Define common pip install flags + PIP_COMMON_FLAGS := --use-pep517 --exists-action s --upgrade --upgrade-strategy eager +endif + +# Define environment-specific pip install flags +ifeq ($(shell uname),Darwin) + PIP_ENV_FLAGS := --break-system-packages +else + PIP_ENV_FLAGS := endif ifeq "$(WAIT)" "" @@ -49,95 +123,222 @@ endif ifeq "$(LOG)" "no" QUIET=@ + ifeq "$(DO_FAIL)" "" + DO_FAIL=$(ECHO) "ok" + endif endif ifeq "$(DO_FAIL)" "" - DO_FAIL=$(ECHO) "ok" + DO_FAIL=$(COMMAND) : +endif + +ifeq "$(RM)" "" + RM=$(COMMAND) rm -f +endif + +ifeq "$(RMDIR)" "" + RMDIR=$(RM)Rd endif -PHONY: must_be_root cleanup +.PHONY: all clean test cleanup init help clean-docs must_be_root must_have_flake must_have_pytest uninstall cleanup-dev-backups + -build: - $(QUIET)$(ECHO) "No need to build. Try make -f Makefile install" - $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile html 2>/dev/null || true +MANIFEST.in: init + $(QUIET)$(ECHO) "include requirements.txt" >"$@" ; + $(QUIET)$(BSMARK) "$@" 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)$(ECHO) "include README.md" >>"$@" ; + $(QUIET)$(ECHO) "include LICENSE.md" >>"$@" ; + $(QUIET)$(ECHO) "include CHANGES.md" >>"$@" ; + $(QUIET)$(ECHO) "include HISTORY.md" >>"$@" ; + $(QUIET)$(ECHO) "recursive-include . *.txt" >>"$@" ; + $(QUIET)$(ECHO) "exclude .gitignore" >>"$@" ; + $(QUIET)$(ECHO) "exclude .deepsource.toml" >>"$@" ; + $(QUIET)$(ECHO) "exclude .*.ini" >>"$@" ; + $(QUIET)$(ECHO) "exclude .*.yml" >>"$@" ; + $(QUIET)$(ECHO) "exclude .*.yaml" >>"$@" ; + $(QUIET)$(ECHO) "global-exclude .git" >>"$@" ; + $(QUIET)$(ECHO) "global-exclude codecov_env" >>"$@" ; + $(QUIET)$(ECHO) "global-exclude .DS_Store" >>"$@" ; + $(QUIET)$(ECHO) "prune .gitattributes" >>"$@" ; + $(QUIET)$(ECHO) "prune test-reports" >>"$@" ; + $(QUIET)$(ECHO) "prune .github" >>"$@" ; + $(QUIET)$(ECHO) "prune .circleci" >>"$@" ; + +build: init ./setup.py MANIFEST.in + $(QUIET)$(PYTHON) -W ignore -m build --sdist --wheel --no-isolation ./ || $(QUIET)$(PYTHON) -W ignore -m build ./ ; + $(QUITE)$(WAIT) + $(QUIET)$(ECHO) "build DONE." init: + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) "pip>=24.3.1" "setuptools>=75.0" "wheel>=0.44" "build>=1.1.1" 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r requirements.txt 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." -install: must_be_root - $(QUIET)python3 -m pip install "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" +install: init build must_be_root + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -e "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" $(QUITE)$(WAIT) $(QUIET)$(ECHO) "$@: Done." -user-install: - $(QUIET)python3 -m pip install --user "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" +user-install: build + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user "pip>=24.3.1" "setuptools>=75.0" "wheel>=0.44" "build>=1.1.1" 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user -r "https://raw.githubusercontent.com/reactive-firewall/python-repo/stable/requirements.txt" 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user -e "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" $(QUITE)$(WAIT) $(QUIET)$(ECHO) "$@: Done." uninstall: - $(QUITE)python3 -m pip uninstall pythonrepo || true + $(QUIET)$(PYTHON) -m pip uninstall --use-pep517 $(PIP_ENV_FLAGS) --no-input -y pythonrepo 2>$(ERROR_LOG_PATH) || : $(QUITE)$(WAIT) $(QUIET)$(ECHO) "$@: Done." -test-reports: - $(QUIET)mkdir test-reports 2>/dev/null >/dev/null || true ; +legacy-purge: clean uninstall + $(QUIET)$(PYTHON) -W ignore ./setup.py uninstall 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(PYTHON) -W ignore ./setup.py clean 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./build/ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./dist/ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./.eggs/ 2>$(ERROR_LOG_PATH) || : + +purge: legacy-purge + $(QUIET)$(RM) ./cc-test-reporter 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./test-reports/*.xml 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." -purge: clean uninstall - $(QUIET)python3 -m pip uninstall pythonrepo && python -m pip uninstall pythonrepo || true +test-reports: + $(QUIET)mkdir $(INST_OPTS) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)$(BSMARK) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(ECHO) "$@: Done." -test: cleanup - $(QUIET)coverage run -p --source=pythonrepo -m unittest discover --verbose -s ./tests -t ./ || python3 -m unittest discover --verbose -s ./tests -t ./ || python -m unittest discover --verbose -s ./tests -t ./ || DO_FAIL=exit 2 ; - $(QUIET)coverage combine 2>/dev/null || true - $(QUIET)coverage report -m --include=pythonrepo* 2>/dev/null || true - $(QUIET)$(DO_FAIL); +test-reqs: test-reports init + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r tests/requirements.txt 2>$(ERROR_LOG_PATH) || true + +just-test: cleanup + $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s ./tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s ./tests -t ./ || DO_FAIL="exit 2" ; + $(QUITE)$(WAIT) ; + $(QUIET)$(DO_FAIL) ; + +test: just-test + $(QUIET)$(DO_FAIL) ; + $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(ECHO) "$@: Done." test-tox: cleanup $(QUIET)tox -v -- || tail -n 500 .tox/py*/log/py*.log 2>/dev/null $(QUIET)$(ECHO) "$@: Done." -test-pytest: cleanup test-reports - $(QUIET)python3 -m pytest --junitxml=test-reports/junit.xml -v tests || python -m pytest --junitxml=test-reports/junit.xml -v tests +test-pytest: cleanup MANIFEST.in must_have_pytest test-reports + $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=. || DO_FAIL="exit 2" ; + $(QUITE)$(WAIT) ; + $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." test-style: cleanup - $(QUIET)python3 -m flake8 --ignore=W191,W391 --max-line-length=100 --verbose --count --config=.flake8.ini --show-source || DO_FAIL="exit 2" ; + $(QUIET)$(PYTHON) -m flake8 --ignore=W191,W391 --max-line-length=100 --verbose --count --config=.flake8.ini --show-source || DO_FAIL="exit 2" ; + $(QUIET)tests/check_cc_lines 2>/dev/null || true $(QUIET)tests/check_spelling 2>/dev/null || true $(QUIET)$(ECHO) "$@: Done." -cleanup: - $(QUIET)rm -f tests/*.pyc 2>/dev/null || true - $(QUIET)rm -f tests/*~ 2>/dev/null || true - $(QUIET)rm -Rf tests/__pycache__ 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*.pyc 2>/dev/null || true - $(QUIET)rm -Rf pythonrepo/__pycache__ 2>/dev/null || true - $(QUIET)rm -Rf pythonrepo/*/__pycache__ 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*~ 2>/dev/null || true - $(QUIET)rm -f *.pyc 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*/*.pyc 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*/*~ 2>/dev/null || true - $(QUIET)rm -f *.DS_Store 2>/dev/null || true - $(QUIET)rm -Rf .pytest_cache/ 2>/dev/null || true - $(QUIET)rmdir ./test-reports/ 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*.DS_Store 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*/*.DS_Store 2>/dev/null || true - $(QUIET)rm -f pythonrepo.egg-info/* 2>/dev/null || true - $(QUIET)rmdir pythonrepo.egg-info 2>/dev/null || true - $(QUIET)rm -f ./*/*~ 2>/dev/null || true - $(QUIET)rm -f ./*~ 2>/dev/null || true - $(QUIET)coverage erase 2>/dev/null || true - $(QUIET)rm -f ./.coverage 2>/dev/null || true - $(QUIET)rm -f ./coverage*.xml 2>/dev/null || true - $(QUIET)rm -f ./sitecustomize.py 2>/dev/null || true - $(QUIET)rm -f ./.*~ 2>/dev/null || true - $(QUIET)rm -Rf ./.tox/ 2>/dev/null || true - -clean: cleanup - $(QUIET)rm -f test-results/junit.xml 2>/dev/null || true - $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile clean || true - $(QUIET)$(ECHO) "$@: Done." +must_have_flake: + $(QUIET)runner=`$(PYTHON) -m pip freeze --all | grep --count -oF flake` ; \ + if test $$runner -le 0 ; then $(ECHO) "No Linter found for test." ; exit 126 ; fi + +must_have_pytest: init + $(QUIET)runner=`$(PYTHON) -m pip freeze --all | grep --count -oF pytest` ; \ + if test $$runner -le 0 ; then $(ECHO) "No python framework (pytest) found for test." ; exit 126 ; fi + +cleanup-dev-backups:: + $(QUIET)$(RM) ./*/*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./.*/*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./**/*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./.*~ 2>$(ERROR_LOG_PATH) || : + +cleanup-mac-dir-store:: + $(QUIET)$(RM) ./.DS_Store 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/.DS_Store 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/.DS_Store 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/**/.DS_Store 2>$(ERROR_LOG_PATH) || : + +cleanup-py-caches: cleanup-dev-backups cleanup-mac-dir-store + $(QUIET)$(RM) ./*.pyc 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/*.pyc 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/__pycache__/* 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/*/*.pyc 2>$(ERROR_LOG_PATH) || : + +cleanup-py-cache-dirs: cleanup-py-caches + $(QUIET)$(RMDIR) ./tests/__pycache__ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./*/__pycache__ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./*/*/__pycache__ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./__pycache__ 2>$(ERROR_LOG_PATH) || : + +cleanup-hypothesis:: + $(QUIET)$(RM) ./.hypothesis/**/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./.hypothesis/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./.hypothesis/ 2>$(ERROR_LOG_PATH) || true + +cleanup-tests: cleanup-hypothesis cleanup-py-cache-dirs cleanup-py-caches + $(QUIET)$(RM) ./test_env/**/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./test_env/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./test_env/ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) .pytest_cache/ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./.tox/ 2>$(ERROR_LOG_PATH) || true + +cleanup-pythonrepo: cleanup-py-cache-dirs cleanup-py-caches + $(QUIET)$(RM) pythonrepo/*.pyc 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) pythonrepo/*~ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) pythonrepo/__pycache__/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) pythonrepo/*/*.pyc 2>$(ERROR_LOG_PATH) || true + +cleanup-pythonrepo-eggs: cleanup-dev-backups cleanup-mac-dir-store + $(QUIET)$(RM) ./*.egg-info/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./*.egg-info 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) .eggs 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./.eggs/ 2>$(ERROR_LOG_PATH) || true + +cleanup-src-dir: cleanup-dev-backups cleanup-mac-dir-store + $(QUIET)$(RM) ./src/**/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./src/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./src/ 2>$(ERROR_LOG_PATH) || true + +cleanup: cleanup-tests cleanup-pythonrepo cleanup-pythonrepo-eggs cleanup-src-dir + $(QUIET)$(RM) ./.coverage 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./coverage*.xml 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./sitecustomize.py 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(WAIT) ; + +build-docs: ./docs/ ./docs/Makefile docs-reqs + $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile html 2>$(ERROR_LOG_PATH) || DO_FAIL="exit 2" ; + $(QUIET)$(WAIT) ; + $(QUIET)mkdir $(INST_OPTS) ./docs/www 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || : ; + $(QUIET)$(BSMARK) ./docs/www 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || : ; + $(QUIET)$(WAIT) ; + $(QUIET)cp -fRp ./docs/_build/ ./docs/www/ 2>$(ERROR_LOG_PATH) || DO_FAIL="exit 35" ; + $(QUIET)$(WAIT) ; + $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile clean 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(WAIT) ; + $(QUIET)$(ECHO) "Documentation should be in docs/www/html/" + $(QUIET)$(DO_FAIL) ; + +clean-docs: ./docs/ ./docs/Makefile + $(QUIET)$(RM) ./docs/www/* 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(RMDIR) ./docs/www/ 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile clean 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(WAIT) ; + +./docs/: + $(QUIET) : ; + +./docs/Makefile: ./docs/ + $(QUIET)$(WAIT) ; + +clean: clean-docs cleanup + $(QUIET)$(ECHO) "Cleaning Up." + $(QUIET)$(COVERAGE) erase 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./test-results/junit.xml 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./MANIFEST.in 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(ECHO) "All clean." must_be_root: $(QUIET)runner=`whoami` ; \ diff --git a/pyproject.tomal b/pyproject.tomal new file mode 100644 index 0000000..e95f439 --- /dev/null +++ b/pyproject.tomal @@ -0,0 +1,12 @@ +[build-system] +requires = ["setuptools>=75.0", "build>=1.2.1", "wheel>=0.44"] +build-backend = "setuptools.build_meta" + +[pytest.enabler.flake8] +addopts = "--flake8" + +[pytest.enabler.doctest] +addopts = "--doctest-glob=**/*.py --doctest-modules" + +[pytest.enabler.cov] +addopts = "--cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml" diff --git a/setup.cfg b/setup.cfg index 7487dc8..603be15 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,50 +12,41 @@ classifiers = Development Status :: 4 - Beta Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux - Programming Language :: Python + License :: OSI Approved :: MIT License Programming Language :: Python :: 3 - Programming Language :: Python :: 3.14 - Programming Language :: Python :: 3.13 Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.4 - Programming Language :: Python :: 3.3 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 :: Only Topic :: Software Development :: Libraries :: Python Modules - Topic :: Security + Topic :: System :: Networking license = MIT license_files = - LICENSE[.md]* + LICENSE.md platform = any project_urls = - Bug Tracker = https://github.com/reactive-firewall/python-repo/issues + "Bug Tracker" = https://github.com/reactive-firewall/python-repo/issues License = https://github.com/reactive-firewall/python-repo/LICENSE.md + Documentation = https://reactive-firewallpython-repo.readthedocs.io/en/stable + Repository = https://github.com/reactive-firewall/python-repo.git [bdist_rpm] url = https://github.com/reactive-firewall/python-repo.git [bdist_wheel] -universal=1 - -[files] -packages = pythonrepo +universal=0 [options] -zip_safe = true +zip_safe = false py_modules = pythonrepo test_suite = tests -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.15.* +python_requires = >=3.9.20 setup_requires = - setuptools>=45.0.0 - wheel>=0.37.0 - build>=1.1.1 + setuptools>=75.0.0 + wheel>=0.44.0 + build>=1.2.1 packages = find: @@ -66,7 +57,7 @@ where = tests/ *.py include = - pythonrepo + pythonrepo.py exclude = docs tests @@ -74,16 +65,19 @@ exclude = [options.extras_require] testing = # upstream - pytest >= 7 + pytest >= 7.4 pytest-checkdocs >= 2.4 - pytest-flake8 - coverage >= 6.3 - pytest-cov >= 4.0.0; \ - # coverage seems to make PyPy extremely slow - python_implementation != "PyPy" + pytest-flake8 >= 1.0.7 + coverage >= 7 pytest-enabler >= 1.0.1 # local flake8 >= 5.0 - virtualenv >= 15.0.1 - wheel >= 0.37.0 - pip >= 21.0 + virtualenv >= 20.26.5 + wheel >= 0.44.0 + pip >= 22.0 + pytest-cov >= 4.0.0; \ + # coverage seems to make PyPy extremely slow + python_implementation != "PyPy" + +[tool:pytest] +addopts = --doctest-glob=**/*.py --doctest-modules --cov= --cov-append --cov-report=xml diff --git a/setup.py b/setup.py index e852299..ce76a51 100755 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ def readFile(filename): str("""Topic :: Software Development :: Libraries :: Python Modules""") ] except Exception: - class_tags = str("""Development Status :: 4 - Beta""") + class_tags = [str("""Development Status :: 4 - Beta""")] setup( name=conf_dict["""metadata"""]["""name"""], diff --git a/tests/check_cc_lines b/tests/check_cc_lines new file mode 100755 index 0000000..d8b1cbe --- /dev/null +++ b/tests/check_cc_lines @@ -0,0 +1,166 @@ +#! /bin/bash +# Disclaimer of Warranties. +# A. YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT, TO THE EXTENT PERMITTED BY +# APPLICABLE LAW, USE OF THIS SHELL SCRIPT AND ANY SERVICES PERFORMED +# BY OR ACCESSED THROUGH THIS SHELL SCRIPT IS AT YOUR SOLE RISK AND +# THAT THE ENTIRE RISK AS TO SATISFACTORY QUALITY, PERFORMANCE, ACCURACY AND +# EFFORT IS WITH YOU. +# +# B. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THIS SHELL SCRIPT +# AND SERVICES ARE PROVIDED "AS IS" AND “AS AVAILABLE”, WITH ALL FAULTS AND +# WITHOUT WARRANTY OF ANY KIND, AND THE AUTHOR OF THIS SHELL SCRIPT'S LICENSORS +# (COLLECTIVELY REFERRED TO AS "THE AUTHOR" FOR THE PURPOSES OF THIS DISCLAIMER) +# HEREBY DISCLAIM ALL WARRANTIES AND CONDITIONS WITH RESPECT TO THIS SHELL SCRIPT +# SOFTWARE AND SERVICES, EITHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES AND/OR CONDITIONS OF +# MERCHANTABILITY, SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE, +# ACCURACY, QUIET ENJOYMENT, AND NON-INFRINGEMENT OF THIRD PARTY RIGHTS. +# +# C. THE AUTHOR DOES NOT WARRANT AGAINST INTERFERENCE WITH YOUR ENJOYMENT OF THE +# THE AUTHOR's SOFTWARE AND SERVICES, THAT THE FUNCTIONS CONTAINED IN, OR +# SERVICES PERFORMED OR PROVIDED BY, THIS SHELL SCRIPT WILL MEET YOUR +# REQUIREMENTS, THAT THE OPERATION OF THIS SHELL SCRIPT OR SERVICES WILL +# BE UNINTERRUPTED OR ERROR-FREE, THAT ANY SERVICES WILL CONTINUE TO BE MADE +# AVAILABLE, THAT THIS SHELL SCRIPT OR SERVICES WILL BE COMPATIBLE OR +# WORK WITH ANY THIRD PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES, +# OR THAT DEFECTS IN THIS SHELL SCRIPT OR SERVICES WILL BE CORRECTED. +# INSTALLATION OF THIS THE AUTHOR SOFTWARE MAY AFFECT THE USABILITY OF THIRD +# PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES. +# +# D. YOU FURTHER ACKNOWLEDGE THAT THIS SHELL SCRIPT AND SERVICES ARE NOT +# INTENDED OR SUITABLE FOR USE IN SITUATIONS OR ENVIRONMENTS WHERE THE FAILURE +# OR TIME DELAYS OF, OR ERRORS OR INACCURACIES IN, THE CONTENT, DATA OR +# INFORMATION PROVIDED BY THIS SHELL SCRIPT OR SERVICES COULD LEAD TO +# DEATH, PERSONAL INJURY, OR SEVERE PHYSICAL OR ENVIRONMENTAL DAMAGE, +# INCLUDING WITHOUT LIMITATION THE OPERATION OF NUCLEAR FACILITIES, AIRCRAFT +# NAVIGATION OR COMMUNICATION SYSTEMS, AIR TRAFFIC CONTROL, LIFE SUPPORT OR +# WEAPONS SYSTEMS. +# +# E. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY THE AUTHOR +# SHALL CREATE A WARRANTY. SHOULD THIS SHELL SCRIPT OR SERVICES PROVE DEFECTIVE, +# YOU ASSUME THE ENTIRE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. +# +# Limitation of Liability. +# F. TO THE EXTENT NOT PROHIBITED BY APPLICABLE LAW, IN NO EVENT SHALL THE AUTHOR +# BE LIABLE FOR PERSONAL INJURY, OR ANY INCIDENTAL, SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES WHATSOEVER, INCLUDING, WITHOUT LIMITATION, DAMAGES +# FOR LOSS OF PROFITS, CORRUPTION OR LOSS OF DATA, FAILURE TO TRANSMIT OR +# RECEIVE ANY DATA OR INFORMATION, BUSINESS INTERRUPTION OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES, ARISING OUT OF OR RELATED TO YOUR USE OR +# INABILITY TO USE THIS SHELL SCRIPT OR SERVICES OR ANY THIRD PARTY +# SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH THIS SHELL SCRIPT OR +# SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY OF LIABILITY (CONTRACT, +# TORT OR OTHERWISE) AND EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION +# OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF INCIDENTAL OR +# CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU. In no event +# shall THE AUTHOR's total liability to you for all damages (other than as may +# be required by applicable law in cases involving personal injury) exceed +# the amount of five dollars ($5.00). The foregoing limitations will apply +# even if the above stated remedy fails of its essential purpose. +################################################################################ + +ulimit -t 600 +PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" +umask 137 + +LOCK_FILE="${TMPDIR:-/tmp}/cc_line_test_script_lock" +EXIT_CODE=1 + +# Function to check if a command exists. + +# USAGE: +# ~$ check_command CMD +# Arguments: +# CMD (Required) -- Name of the command to check +# Results: +# exits 64 -- missing required argument +# exits 126 -- check complete and has failed, can not find given command. +# returns successful -- check complete and command found to be executable. +function check_command() { + test -z "$1" && { printf "%s\n" "Error: command name is required to check for existence." >&2 ; exit 64 ; } ; + local cmd="$1" ; + test -x "$(command -v ${cmd})" || { printf "%s\n" "Error: Required command '$cmd' is not found." >&2 ; exit 126 ; } ; +} # end check_command() +# propagate/export function to sub-shells +export -f check_command + +# Set up CEP-5 shlock helper +hash -p ./.github/tool_shlock_helper.sh shlock || { printf "%s\n" "Error: Failed to register shlock helper. CEP-5 locking will not work." >&2 ; exit 78 ; } ; + +# Check required commands +check_command grep ; +check_command curl ; +check_command find ; +check_command git ; +check_command shlock ; + + +function cleanup() { + rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; + hash -d shlock 2>/dev/null > /dev/null || true ; +} + +if [[ ( $(shlock -f ${LOCK_FILE} -p $$ ) -eq 0 ) ]] ; then + EXIT_CODE=0 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 + # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 + #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true || true ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 +else + printf "%s\n" "Check for Copyright lines already in progress by "`head ${LOCK_FILE}` ; + false ; + exit 126 ; +fi + +# this is how test files are found: +_TEST_FILE_VALIDATOR="echo " + +# THIS IS THE ACTUAL TEST +if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); then + if [ -z "${_TEST_ROOT_DIR}" ]; then + _TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) + fi +else + printf "\t%s\n" "FAIL: missing valid repository or source structure" >&2 + EXIT_CODE=40 +fi + +_TEST_YEAR=$(date -j "+%C%y" 2>/dev/null ;) + +for _TEST_DOC in $(find ${_TEST_ROOT_DIR} \( -not -ipath "*.github/*" \) -a \( -iname '*.py' -o -iname '*.txt' -o -iname '*.md' \) -a -print0 2>&1 | xargs -0 -I{} ${_TEST_FILE_VALIDATOR} "{}" ; wait ;) ; do + if [[ ($(grep -cF 'Disclaimer' "${_TEST_DOC}" 2>&1 || : ;) -ne 0) ]] ; then + printf "%s\n" "SKIP: ${_TEST_DOC} is disclaimed." ; + EXIT_CODE=126; + else + if [[ ($(grep -cF "Copyright" "${_TEST_DOC}" 2>&1 || : ;) -le 0) ]] ; then + printf "%s\n" "FAIL: ${_TEST_DOC} is missing a copyright line" >&2 ; + EXIT_CODE=127 + fi + if [[ ( $(grep -F "Copyright" "${_TEST_DOC}" 2>&1 | grep -coF "Copyright (c)" 2>&1) -le 0) ]] ; then + printf "%s\n" "SKIP: ${_TEST_DOC} is missing a valid copyright line begining with \"Copyright (c)\"" ; + fi + if [[ ( $(grep -F "Copyright (c)" "${_TEST_DOC}" 2>&1 | grep -oE "\d+(-\d+)?" 2>&1 | grep -oE "\d{3,}$" | sort -n | tail -n1) -lt ${_TEST_YEAR}) ]] ; then + printf "%s\n" "WARN: ${_TEST_DOC} is out of date without a current copyright (year)" >&2 ; + fi + if [[ ( ${EXIT_CODE} -ne 0 ) ]] ; then + case "$EXIT_CODE" in + 0|126) true ;; + *) printf "%s\n" "SKIP: Unclassified issue with '${_TEST_DOC}'" ;; + esac + fi + fi +done + +unset _TEST_ROOT_DIR 2>/dev/null || true ; +unset _TEST_DOC 2>/dev/null || true ; +unset _TEST_YEAR 2>/dev/null || true ; + +rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; + +# goodbye +exit ${EXIT_CODE:-255} ; diff --git a/tests/check_spelling b/tests/check_spelling index a6bc29c..15b0e77 100755 --- a/tests/check_spelling +++ b/tests/check_spelling @@ -1,19 +1,214 @@ #! /bin/bash +# Disclaimer of Warranties. +# A. YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT, TO THE EXTENT PERMITTED BY +# APPLICABLE LAW, USE OF THIS SHELL SCRIPT AND ANY SERVICES PERFORMED +# BY OR ACCESSED THROUGH THIS SHELL SCRIPT IS AT YOUR SOLE RISK AND +# THAT THE ENTIRE RISK AS TO SATISFACTORY QUALITY, PERFORMANCE, ACCURACY AND +# EFFORT IS WITH YOU. +# +# B. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THIS SHELL SCRIPT +# AND SERVICES ARE PROVIDED "AS IS" AND "AS AVAILABLE", WITH ALL FAULTS AND +# WITHOUT WARRANTY OF ANY KIND, AND THE AUTHOR OF THIS SHELL SCRIPT'S LICENSORS +# (COLLECTIVELY REFERRED TO AS "THE AUTHOR" FOR THE PURPOSES OF THIS DISCLAIMER) +# HEREBY DISCLAIM ALL WARRANTIES AND CONDITIONS WITH RESPECT TO THIS SHELL SCRIPT +# SOFTWARE AND SERVICES, EITHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES AND/OR CONDITIONS OF +# MERCHANTABILITY, SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE, +# ACCURACY, QUIET ENJOYMENT, AND NON-INFRINGEMENT OF THIRD PARTY RIGHTS. +# +# C. THE AUTHOR DOES NOT WARRANT AGAINST INTERFERENCE WITH YOUR ENJOYMENT OF THE +# THE AUTHOR's SOFTWARE AND SERVICES, THAT THE FUNCTIONS CONTAINED IN, OR +# SERVICES PERFORMED OR PROVIDED BY, THIS SHELL SCRIPT WILL MEET YOUR +# REQUIREMENTS, THAT THE OPERATION OF THIS SHELL SCRIPT OR SERVICES WILL +# BE UNINTERRUPTED OR ERROR-FREE, THAT ANY SERVICES WILL CONTINUE TO BE MADE +# AVAILABLE, THAT THIS SHELL SCRIPT OR SERVICES WILL BE COMPATIBLE OR +# WORK WITH ANY THIRD PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES, +# OR THAT DEFECTS IN THIS SHELL SCRIPT OR SERVICES WILL BE CORRECTED. +# INSTALLATION OF THIS THE AUTHOR SOFTWARE MAY AFFECT THE USABILITY OF THIRD +# PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES. +# +# D. YOU FURTHER ACKNOWLEDGE THAT THIS SHELL SCRIPT AND SERVICES ARE NOT +# INTENDED OR SUITABLE FOR USE IN SITUATIONS OR ENVIRONMENTS WHERE THE FAILURE +# OR TIME DELAYS OF, OR ERRORS OR INACCURACIES IN, THE CONTENT, DATA OR +# INFORMATION PROVIDED BY THIS SHELL SCRIPT OR SERVICES COULD LEAD TO +# DEATH, PERSONAL INJURY, OR SEVERE PHYSICAL OR ENVIRONMENTAL DAMAGE, +# INCLUDING WITHOUT LIMITATION THE OPERATION OF NUCLEAR FACILITIES, AIRCRAFT +# NAVIGATION OR COMMUNICATION SYSTEMS, AIR TRAFFIC CONTROL, LIFE SUPPORT OR +# WEAPONS SYSTEMS. +# +# E. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY THE AUTHOR +# SHALL CREATE A WARRANTY. SHOULD THIS SHELL SCRIPT OR SERVICES PROVE DEFECTIVE, +# YOU ASSUME THE ENTIRE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. +# +# Limitation of Liability. +# F. TO THE EXTENT NOT PROHIBITED BY APPLICABLE LAW, IN NO EVENT SHALL THE AUTHOR +# BE LIABLE FOR PERSONAL INJURY, OR ANY INCIDENTAL, SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES WHATSOEVER, INCLUDING, WITHOUT LIMITATION, DAMAGES +# FOR LOSS OF PROFITS, CORRUPTION OR LOSS OF DATA, FAILURE TO TRANSMIT OR +# RECEIVE ANY DATA OR INFORMATION, BUSINESS INTERRUPTION OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES, ARISING OUT OF OR RELATED TO YOUR USE OR +# INABILITY TO USE THIS SHELL SCRIPT OR SERVICES OR ANY THIRD PARTY +# SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH THIS SHELL SCRIPT OR +# SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY OF LIABILITY (CONTRACT, +# TORT OR OTHERWISE) AND EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION +# OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF INCIDENTAL OR +# CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU. In no event +# shall THE AUTHOR's total liability to you for all damages (other than as may +# be required by applicable law in cases involving personal injury) exceed +# the amount of five dollars ($5.00). The foregoing limitations will apply +# even if the above stated remedy fails of its essential purpose. +################################################################################ +# +# This script attempts to enforce spell-checking if the codespell tool is available. +# It accomplishes the following tasks: +# 1. Sets up error handling and cleanup mechanisms to ensure proper execution. +# 2. Determines the test root directory based on the project structure. +# 3. lint and check spelling +# 4. cleanup and report +# +# Usage Summary: +# To lint the project without making changes: +# ./tests/check_spelling +# To lint and auto-correct spelling errors: +# ./tests/check_spelling --fix +# +# Exit Code Summary: +# The script uses the EXIT_CODE variable to track exit conditions: +# - 0: Successful execution. +# - 1: General failure. +# - 2: Coverage combine or XML generation failed. +# - 3: git ls-tree command failed. +# - 40: Missing valid repository or source structure. +# - 126: Script already in progress or command not executable. +# - 129: Received SIGHUP signal. +# - 130: Received SIGINT signal (Ctrl+C). +# - 131: Received SIGQUIT signal. +# - 137: Received SIGABRT signal. +# - 143: Received SIGTERM signal. +# +# The primary goal is to allow linter like spell-checking. -# exit fast if command is missing -test -x /usr/bin/spellintian || exit 0 ; +ulimit -t 600 +# setting the path may break brain-dead CI that uses crazy paths +# PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" +umask 137 -THE_TEMP_FILE="/tmp/swapfile_spellcheck_${RANDOM}.tmp.txt" ; -( (spellintian "${@:-./**/*}" 2>/dev/null | fgrep -v "(duplicate word)" | fgrep " -> ") & (spellintian "${@:-./*}" 2>/dev/null | fgrep -v "(duplicate word)" | fgrep " -> ") & (spellintian "${@:-./**/**/*}" 2>/dev/null | fgrep -v "(duplicate word)" | fgrep " -> ") ) | sort -h | uniq | tee -a ${THE_TEMP_FILE:-/dev/null} ; -wait ; -THECOUNT=$( (wc -l ${THE_TEMP_FILE} 2>/dev/null || echo 0) | cut -d\ -f 1 ) ; -EXIT_CODE=${THECOUNT} ; -if [[ ("${THECOUNT}" -le 1) ]] ; then - EXIT_CODE=0 ; - echo "OK: Found no detected spelling errors." ; +# force utf-8 for spelling +export LC_CTYPE="${LC_CTYPE:-en_US.UTF-8}" + +LOCK_FILE="${TMPDIR:-/tmp}/org.pak.multicast.spell-check-shell" +EXIT_CODE=1 + +# Function to check if a command exists. + +# USAGE: +# ~$ check_command CMD +# Arguments: +# CMD (Required) -- Name of the command to check +# Results: +# exits 64 -- missing required argument +# exits 126 -- check complete and has failed, can not find given command. +# returns successful -- check complete and command found to be executable. +function check_command() { + test -z "$1" && { printf "%s\n" "Error: command name is required to check for existence." >&2 ; exit 64 ; } ; + local cmd="$1" ; + test -x "$(command -v ${cmd})" || { printf "%s\n" "Error: Required command '$cmd' is not found." >&2 ; exit 126 ; } ; +} # end check_command() +# propagate/export function to sub-shells +export -f check_command + +# Set up CEP-5 shlock helper +hash -p ./.github/tool_shlock_helper.sh shlock || { printf "%s\n" "Error: Failed to register shlock helper. CEP-5 locking will not work." >&2 ; exit 78 ; } ; + +# Check required commands +check_command grep ; +check_command python3 ; +check_command codespell ; +check_command git ; +check_command shlock ; + + +SCRIPT_FILE="tests/check_spelling" + +# Set codespell options +CODESPELL_OPTIONS="--quiet-level=4 --builtin clear,rare,code -L assertIn" + +function cleanup() { + rm -f ${LOCK_FILE} 2>/dev/null || : ; wait ; + # unset when done + unset LOCK_FILE 2>/dev/null || : ; + hash -d shlock 2>/dev/null || : ; +} + +if [[ ( $(shlock -f ${LOCK_FILE} -p $$ ) -eq 0 ) ]] ; then + EXIT_CODE=0 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 + # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 + #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true || true ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 else - echo "FAIL: Found ${THECOUNT:-many} spelling errors." ; + # shellcheck disable=SC2046 + printf "\t%s\n" "Check Setup Scripts Tests Coverage already in progress by "$(head "${LOCK_FILE}") >&2 ; + exit 126 ; fi -rm -f ${THE_TEMP_FILE} 2>/dev/null >> /dev/null || true ; + +function report_summary() { + printf "::group::%s\n" "Results" ; + # Improved reporting based on EXIT_CODE + case "${EXIT_CODE}" in + 0) printf "::notice title=OK::%s\n" "OK: Found no detected spelling errors." ;; + 1) printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=CHECK-SPELLING::%s\n" "FAIL: General failure during script execution." >&2 ;; + 3) printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=CONFIGURATION::%s\n" "FAIL: Gathering repostory's requirements failed." >&2 ;; # git ls-tree command failed + 126) printf "::warning file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=SKIPPED::%s\n" "SKIP: Unable to continue script execution." >&2 ;; + *) printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=FAILED::%s\n" "FAIL: Detected spelling errors." >&2 ;; + esac + printf "::endgroup::\n" ; +} + +# this is how test files are found: + +# THIS IS THE ACTUAL TEST +_TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) ; +if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); then + if [ -z "${_TEST_ROOT_DIR}" ]; then + _TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) + fi + printf "::debug::%s\n" "Found ${_TEST_ROOT_DIR} ..." ; + else + printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}::%s\n" "FAIL: missing valid repository or source structure" >&2 + EXIT_CODE=40 +fi + +# Get a list of files to check using git ls-tree with filtering +FILES_TO_CHECK=$(git ls-tree -r --full-tree --name-only HEAD -- *.md *.py *.txt **/*.md **/*.txt **/*.py 2>/dev/null || EXIT_CODE=3) + +# Enable auto-correction if '--fix' argument is provided +if [[ "$1" == "--fix" ]]; then + CODESPELL_OPTIONS="--write-changes --interactive 2 ${CODESPELL_OPTIONS}" + printf "::debug::%s\n" "Auto-correction enabled." +fi + +# THIS IS THE ACTUAL TEST +# Iterate over files and run codespell +for FILE in $FILES_TO_CHECK; do + printf "::group::%s\n" "Checking ${FILE}" ; + { codespell $CODESPELL_OPTIONS "${FILE}" || EXIT_CODE=$? ;} 2>/dev/null ; wait ; + printf "::endgroup::\n" ; +done + +# cleaning up and reporting +report_summary + +cleanup || rm -f ${LOCK_FILE} 2>/dev/null || : ; + +# unset when done +unset _TEST_ROOT_DIR 2>/dev/null || : ; +unset CODESPELL_OPTIONS 2>/dev/null || : ; + wait ; -exit ${EXIT_CODE:255} ; +exit ${EXIT_CODE:-255} ; diff --git a/tests/context.py b/tests/context.py index 3552f41..bf4b164 100644 --- a/tests/context.py +++ b/tests/context.py @@ -21,18 +21,58 @@ """This is pythonrepo testing module Template.""" +__module__ = """tests""" + +__name__ = """tests.context""" # skipcq: PYL-W0622 + +__doc__ = """ + + Robust imports: These statements import the entire "pythonrepo" module, + allowing access to all its functionalities within the test environment. + This can be flagged as an intentional + [cyclic-import](https://pylint.pycqa.org/en/latest/user_guide/messages/refactor/cyclic-import.html) + warning. + + Context for Testing. + + Meta Tests - Fixtures: + + Test fixtures by importing test context. + + >>> import tests.context as context + >>> + + >>> from context import pythonrepo as _pythonrepo + >>> + + >>> from context import profiling as _profiling + >>> + +""" + try: import sys - import os - if 'pythonrepo' in __file__: - sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -except Exception as badErr: - baton = ImportError(badErr, str("[CWE-758] Test module failed completely.")) - baton.module = __module__ - baton.path = __file__ - baton.__cause__ = badErr - raise baton + if not hasattr(sys, 'modules') or not sys.modules: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] sys.modules is not available or empty.") from None +except ImportError as err: + raise ImportError("[CWE-440] Unable to import sys module.") from err + +try: + if 'os' not in sys.modules: + import os + else: # pragma: no branch + os = sys.modules["""os"""] +except ImportError as err: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] OS Failed to import.") from err + +try: + if 'tests.profiling' not in sys.modules: + import tests.profiling as profiling + else: # pragma: no branch + profiling = sys.modules["""tests.profiling"""] +except ImportError as err: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] profiling Failed to import.") from err try: import pythonrepo as pythonrepo # skipcq: PYL-C0414 diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index c84b235..77867dd 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -60,7 +60,7 @@ # even if the above stated remedy fails of its essential purpose. ################################################################################ -ulimit -t 1200 +ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" LANG=${LANG:-"en_US"} LC_ALL="${LANG:1:5}.utf-8" @@ -76,29 +76,60 @@ test -x "$(command -v shlock)" || exit 126 ; test -x "$(command -v gpgv)" || exit 126 ; test -x "$(command -v shasum)" || exit 126 ; -# sorry no windows support here -if [[ $( command uname -s ) == *arwin ]] ; then - CI_OS="darwin" -else - CI_OS="linux" -fi +# Detect the operating system +case "$( command uname -s )" in + *arwin) + CI_OS="darwin" + ;; + Linux) + CI_OS="linux" + ;; + *) + # sorry no windows support here + printf 'Unsupported OS\n' >&2 ; + exit 75 ; # exit code for temporary failure, as per CEP-8 + ;; +esac + +# Detect the architecture +ARCH="$(command uname -m)" +case "$ARCH" in + x86_64) + ARCH="amd64" + ;; + arm64|aarch64|armv8l) + ARCH="arm64" + ;; + armv7l|armv6l) + ARCH="arm" + ;; + *) + # sorry no windows support here + printf 'Unsupported architecture\n' >&2 ; + exit 75 ; + ;; +esac + +# Set the CI prefix +CI_ENV_PREFIX="${CI_OS}-${ARCH}" function cleanup() { - rm -f "${LOCK_FILE}" 2>/dev/null || true ; wait ; - rm -f ./test-reporter-latest-*-amd64.SHA*.sig 2>/dev/null || true ; wait ; - rm -f ./test-reporter-latest-*-amd64.SHA* 2>/dev/null > /dev/null || true ; wait ; - hash -d shlock 2>/dev/null > /dev/null || true ; + rm -f "${LOCK_FILE}" 2>/dev/null || : ; wait ; + rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.SHA*.sig 2>/dev/null || : ; wait ; + rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.SHA* 2>/dev/null > /dev/null || : ; wait ; + unset CI_ENV_PREFIX 2>/dev/null || : + hash -d shlock 2>/dev/null > /dev/null || : ; } if [[ ( $(shlock -f "${LOCK_FILE}" -p $$ ) -eq 0 ) ]] ; then - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 - #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true || true ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 + #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true || : ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 trap 'cleanup ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 else # shellcheck disable=SC2046 @@ -124,19 +155,19 @@ fi # to be SAFELY USED to store Tokens and env vars (update logic as needed) if [[ ( -r ./codecov_env ) ]] ; then # shellcheck disable=SC2086,SC1091 - source ./codecov_env 2>/dev/null || true ; + source ./codecov_env 2>/dev/null || : ; fi ######################### # actual Work starts here ######################### -curl -fLso ./test-reporter-latest-${CI_OS:-linux}-amd64 https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-amd64 ; +curl -fLso ./test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64} https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64} ; for i in 1 256 512 ; do - curl -fLso test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}" ; wait ; - curl -fLso test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig" ; wait ; + curl -fLso test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}" ; wait ; + curl -fLso test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig" ; wait ; # test sha1/sha512 signatures if found and sha256 even if not found - if [[ ( -r test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then + if [[ ( -r test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then if [[ ${i} -eq 1 ]]; then printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" fi @@ -145,27 +176,27 @@ for i in 1 256 512 ; do # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 # OR... # Set CI=true to continue on missing keys - gpg --verify test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} || ${CI} || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig 2>/dev/null ; + gpg --verify test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || ${CI} || EXIT_CODE=126 + rm -vf test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig 2>/dev/null ; fi - shasum -a $i -c --ignore-missing test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} 2>/dev/null ; + shasum -a $i -c --ignore-missing test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || EXIT_CODE=126 + rm -vf test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} 2>/dev/null ; fi done if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then mv -f test-reporter-latest-${CI_OS:-linux}-amd64 ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 - chmod -v 751 ./cc-test-reporter || EXIT_CODE=126 + chmod -v 751 ./cc-test-reporter || EXIT_CODE=77 fi if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - ./cc-test-reporter before-build || EXIT_CODE=10 ; + ./cc-test-reporter before-build || EXIT_CODE=125 ; fi -unset _TEST_ROOT_DIR 2>/dev/null || true ; +unset _TEST_ROOT_DIR 2>/dev/null || : ; -cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || true ; wait ; +cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait ; # goodbye exit ${EXIT_CODE:-255} ; From f40d336beccc57387825877174ad2e4d6e4e4fcc Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 27 Nov 2024 00:40:26 -0800 Subject: [PATCH 02/58] [STYLE] fixed some typos (- WIP #79 -) --- docs/conf.py | 2 +- pythonrepo/pythonrepo.py | 6 +++--- tests/test_usage.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 23e6b33..314411b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,7 @@ '.rst': 'restructuredtext', } -# The encoding of source files. Official sphinx docs reccomend utf-8-sig. +# The encoding of source files. Official sphinx docs recommend utf-8-sig. source_encoding = 'utf-8-sig' # The master toctree document. diff --git a/pythonrepo/pythonrepo.py b/pythonrepo/pythonrepo.py index e20e95a..e785263 100644 --- a/pythonrepo/pythonrepo.py +++ b/pythonrepo/pythonrepo.py @@ -102,7 +102,7 @@ def useTool(tool, *arguments): try: TASK_OPTIONS[tool](arguments) except Exception: - w = str("WARNING - An error occured while") + w = str("WARNING - An error occurred while") w += str("handling the tool. Abort.") print(w) else: @@ -117,13 +117,13 @@ def main(*argv): service_cmd = args.some_task useTool(service_cmd, extra) except Exception: - w = str("WARNING - An error occured while") + w = str("WARNING - An error occurred while") w += str("handling the arguments.") w += str(" Cascading failure.") print(w) sys.exit(2) except Exception: - e = str("CRITICAL - An error occured while handling") + e = str("CRITICAL - An error occurred while handling") e += str("the cascading failure.") print(e) sys.exit(3) diff --git a/tests/test_usage.py b/tests/test_usage.py index b75d253..9af1a7e 100644 --- a/tests/test_usage.py +++ b/tests/test_usage.py @@ -113,7 +113,7 @@ def checkPythonErrors(args=None, stderr=None): taintArgs = buildPythonCommand(args) theOutput = subprocess.check_output(taintArgs, stderr=stderr) if isinstance(theOutput, bytes): - # default to utf8 your milage may vary + # default to utf8 your mileage may vary theOutput = theOutput.decode('utf8') except Exception as err: theOutput = None From 6fc01891e8301fcc547fc55ec6838fb2eb53a646 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Sun, 8 Dec 2024 21:20:04 -0800 Subject: [PATCH 03/58] [CI] Part 2 of many (- WIP PR #82 -) --- .github/workflows/Tests.yml | 157 +++++++++-------------------------- Makefile | 23 +++-- tests/fetch_cc-test-reporter | 78 ++++++++++++++--- 3 files changed, 121 insertions(+), 137 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 16a0104..ec7420b 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -43,7 +43,7 @@ jobs: BOOTSTRAP: - if: ${{ always() }} + if: ${{ !cancelled() }} needs: BUILD runs-on: ubuntu-latest defaults: @@ -54,22 +54,22 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] lang-var: ["de.utf-8", "jp.utf-8"] experimental: [true] include: - python-version: 3.7 lang-var: "en_US.utf-8" - experimental: false + experimental: true - python-version: 3.8 lang-var: "en_US.utf-8" - experimental: false + experimental: true - python-version: 3.9 lang-var: "de.utf-8" - experimental: false + experimental: true - python-version: 3.9 lang-var: "jp.utf-8" - experimental: false + experimental: true - python-version: 3.9 lang-var: "en_US.utf-8" experimental: false @@ -93,7 +93,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Setup dependencies run: | - python -m pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" + python -m pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" pip install -r ./requirements.txt pip install -r ./test-requirements.txt || true - name: Pre-build @@ -126,12 +126,12 @@ jobs: run: | make -j1 -f Makefile purge || true ; make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} shell: bash MATS: - if: ${{ always() }} + if: ${{ !cancelled() }} needs: BUILD runs-on: ubuntu-latest defaults: @@ -140,7 +140,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: ["3.9.20", "3.10", "3.11", "3.12"] env: PYTHON_VERSION: ${{ matrix.python-version }} LANG: "en_US.utf-8" @@ -169,10 +169,9 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} - + if: ${{ !cancelled() }} - COVERAGE-MATS: + COVERAGE: if: ${{ success() }} needs: [BUILD, MATS] runs-on: ${{ matrix.os }} @@ -180,86 +179,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12"] - env: - OS: ${{ matrix.os }} - PYTHON_VERSION: ${{ matrix.python-version }} - LANG: "en_US.utf-8" - COVERAGE_RCFILE: ./.coveragerc - COV_CORE_SOURCE: ./ - COV_CORE_CONFIG: ./.coveragerc - COV_CORE_DATAFILE: ./coverage.xml - CODECLIMATE_REPO_TOKEN: ${{ secrets.CODECLIMATE_TOKEN }} - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - steps: - - uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} - run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" - pip install -r ./requirements.txt - pip install -r ./test-requirements.txt || true - - name: Install code-climate tools for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" - shell: bash - run: if [ "$OS" == "macos-latest" ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; - - name: Install deepsource tools for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" - shell: bash - run: | - if [ "$OS" == "ubuntu-latest" ] ; then (curl https://deepsource.io/cli | sh) || true ; else echo "SKIP deepsource" ; fi ; - - name: Pre-Clean - id: clean - run: make -j1 -f Makefile clean || true ; - - name: Generate Coverage for py3.9 on ${{ matrix.os }} - if: ${{ runner.python-version }} == "3.9" - run: make -f Makefile test ; - - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} - if: ${{ runner.python-version }} != "3.9" - run: make -f Makefile test-pytest ; - - name: Upload Python ${{ matrix.python-version }} coverage to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - directory: . - flags: ${{ matrix.os }},${{ matrix.python-version }} - name: pythonrepo-github-${{ matrix.os }}-${{ matrix.python-version }} - verbose: true - fail_ci_if_error: false - - name: Upload Python ${{ matrix.python-version }} Artifact - uses: actions/upload-artifact@v4 - with: - name: Test-Report-${{ matrix.os }}-${{ matrix.python-version }} - path: ./test-reports/ - if-no-files-found: ignore - - name: code-climate for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" - shell: bash - run: | - if [ "$OS" == "macos-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; - - name: deepsource for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" - shell: bash - run: | - if [ "$OS" == "ubuntu-latest" ] ; then ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; else echo "SKIP deepsource" ; fi ; - - name: Post-Clean - id: post - run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} - - COVERAGE: - if: ${{ success() }} - needs: [BUILD, MATS, COVERAGE-MATS] - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - strategy: - matrix: - os: [ubuntu-latest, macos-13, windows-latest] - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] env: OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} @@ -267,7 +187,8 @@ jobs: COVERAGE_RCFILE: ./.coveragerc COV_CORE_SOURCE: ./ COV_CORE_CONFIG: ./.coveragerc - COV_CORE_DATAFILE: ./coverage.xml + COV_CORE_DATAFILE: ./coverage.xml\ + DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }} CODECLIMATE_REPO_TOKEN: ${{ secrets.CODECLIMATE_TOKEN }} CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} steps: @@ -276,20 +197,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Fix braindead windows ${{ matrix.python-version }} on ${{ matrix.os }} + if: ${{ !cancelled() && runner.os == 'Windows' }} + run: python -m pip install --upgrade pip - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" ; + pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; pip install -r ./requirements.txt ; pip install -r ./test-requirements.txt || true ; - name: Install code-climate tools for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" - shell: bash - run: if [ "$OS" == "ubuntu-latest" ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; - - name: Install deepsource tools for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash - run: | - if [ "$OS" == "ubuntu-latest" ] ; then (curl https://deepsource.io/cli | sh) || true ; else echo "SKIP deepsource" ; fi ; + run: ./tests/fetch_cc-test-reporter || true ; - name: Pre-Clean id: clean run: make -j1 -f Makefile clean || true ; @@ -316,7 +235,7 @@ jobs: path: ./test-reports/ if-no-files-found: ignore - name: code-climate for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash run: | if [ "$OS" == "ubuntu-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; @@ -328,7 +247,7 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} STYLE: if: ${{ success() }} @@ -337,7 +256,7 @@ jobs: timeout-minutes: 10 env: - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.12' LANG: "en_US.utf-8" steps: @@ -345,10 +264,10 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies for python Linters run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" ; + pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; pip install -r ./requirements.txt ; pip install -r ./test-requirements.txt || true ; - name: Pre-Clean @@ -361,7 +280,7 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} INTEGRATION: @@ -392,11 +311,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" ; + pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; pip install -r ./requirements.txt ; pip install -r ./test-requirements.txt || true ; - - name: Install code-climate tools for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" + - name: Install code-climate tools for ${{ matrix.python-version }} on ${{ matrix.os }} + if: ${{ !cancelled() && runner.os != 'Windows' }} run: if [ $OS == macos-latest ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; shell: bash - name: Pre-Clean @@ -430,17 +349,17 @@ jobs: path: ./test-reports/ if-no-files-found: ignore - name: code-climate for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} run: | - if [ "$OS" == "macos-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; + ./cc-test-reporter after-build --exit-code 0 || true ; - name: Post-purge id: post-uninstall run: make -j1 -f Makefile purge || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} - name: Post-Clean id: post-z-end run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} TOX: @@ -450,7 +369,7 @@ jobs: timeout-minutes: 30 env: - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.12' LANG: 'en_US.utf-8' steps: @@ -458,10 +377,10 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies for Tox run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" "tox>=3.0"; + pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" "tox>=3.0"; pip install -r ./requirements.txt ; pip install -r ./test-requirements.txt || true ; - name: Pre-Clean @@ -472,4 +391,4 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} diff --git a/Makefile b/Makefile index 8bf77be..c400d8c 100644 --- a/Makefile +++ b/Makefile @@ -176,14 +176,14 @@ init: install: init build must_be_root $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -e "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" - $(QUITE)$(WAIT) + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "$@: Done." user-install: build $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user "pip>=24.3.1" "setuptools>=75.0" "wheel>=0.44" "build>=1.1.1" 2>$(ERROR_LOG_PATH) || true $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user -r "https://raw.githubusercontent.com/reactive-firewall/python-repo/stable/requirements.txt" 2>$(ERROR_LOG_PATH) || true $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user -e "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" - $(QUITE)$(WAIT) + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "$@: Done." uninstall: @@ -200,6 +200,7 @@ legacy-purge: clean uninstall purge: legacy-purge $(QUIET)$(RM) ./cc-test-reporter 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./ds-cli.sh 2>$(ERROR_LOG_PATH) || : $(QUIET)$(RM) ./test-reports/*.xml 2>$(ERROR_LOG_PATH) || : $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." @@ -209,15 +210,15 @@ test-reports: $(QUIET)$(BSMARK) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(ECHO) "$@: Done." -test-reqs: test-reports init +test-reqs: cc-test-reporter test-reports init $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r tests/requirements.txt 2>$(ERROR_LOG_PATH) || true just-test: cleanup $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s ./tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s ./tests -t ./ || DO_FAIL="exit 2" ; - $(QUITE)$(WAIT) ; + $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; -test: just-test +test: just-test cc-test-reporter $(QUIET)$(DO_FAIL) ; $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; @@ -227,9 +228,11 @@ test-tox: cleanup $(QUIET)tox -v -- || tail -n 500 .tox/py*/log/py*.log 2>/dev/null $(QUIET)$(ECHO) "$@: Done." -test-pytest: cleanup MANIFEST.in must_have_pytest test-reports +test-pytest: cleanup MANIFEST.in cc-test-reporter must_have_pytest test-reports $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=. || DO_FAIL="exit 2" ; - $(QUITE)$(WAIT) ; + $(QUIET)./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml || : ; + $(QUIET)./cc-test-reporter after-build --exit-code 0 -t coverage.py -r $(CC_TEST_REPORTER_ID) || : ; + $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." @@ -239,6 +242,12 @@ test-style: cleanup $(QUIET)tests/check_spelling 2>/dev/null || true $(QUIET)$(ECHO) "$@: Done." +cc-test-reporter: tests/fetch_cc-test-reporter + $(QUIET)tests/fetch_cc-test-reporter ; + $(QUIET)$(WAIT) ; + $(QUIET)$(DO_FAIL) ; + $(QUIET)$(ECHO) "$@: Done." + must_have_flake: $(QUIET)runner=`$(PYTHON) -m pip freeze --all | grep --count -oF flake` ; \ if test $$runner -le 0 ; then $(ECHO) "No Linter found for test." ; exit 126 ; fi diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 77867dd..b9ba9af 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -60,6 +60,13 @@ # even if the above stated remedy fails of its essential purpose. ################################################################################ +# CAVEAT: This Script is for convience and should not be used as a source of truth +# when checking crypto hashes. + + +# to check though: +diff -q <(tail -n 191 "$0" | head -n 189 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; + ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" LANG=${LANG:-"en_US"} @@ -115,9 +122,12 @@ CI_ENV_PREFIX="${CI_OS}-${ARCH}" function cleanup() { rm -f "${LOCK_FILE}" 2>/dev/null || : ; wait ; - rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.SHA*.sig 2>/dev/null || : ; wait ; - rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.SHA* 2>/dev/null > /dev/null || : ; wait ; + rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.sha*.sig 2>/dev/null || : ; wait ; + rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.sha* 2>/dev/null > /dev/null || : ; wait ; + rm -f ./ds-cli.sh.sha* 2>/dev/null > /dev/null || : ; wait ; unset CI_ENV_PREFIX 2>/dev/null || : + unset CI_OS 2>/dev/null || : + unset ARCH 2>/dev/null || : hash -d shlock 2>/dev/null > /dev/null || : ; } @@ -141,19 +151,22 @@ fi # this is how test files are found: # THIS IS THE ACTUAL TEST DIR USED (update _TEST_ROOT_DIR as needed) -_TEST_ROOT_DIR="./" ; -if [[ -d ./.git ]] ; then - _TEST_ROOT_DIR="./" ; -elif [[ -d ./tests ]] ; then - _TEST_ROOT_DIR="./" ; +_TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) ; +if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); then + if [ -z "${_TEST_ROOT_DIR}" ]; then + _TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) + fi + printf "::debug::%s\n" "Found ${_TEST_ROOT_DIR} ..." ; else - printf "%s\n" "FAIL: missing valid folder or file" - EXIT_CODE=1 + printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}::%s\n" \ + "FAIL: missing valid repository or source structure" >&2 ; + EXIT_CODE=40 fi # This File MUST BE GIT-IGNORED # to be SAFELY USED to store Tokens and env vars (update logic as needed) if [[ ( -r ./codecov_env ) ]] ; then + printf "::debug::%s\n" "Found codecov_env ..." ; # shellcheck disable=SC2086,SC1091 source ./codecov_env 2>/dev/null || : ; fi @@ -171,7 +184,7 @@ for i in 1 256 512 ; do if [[ ${i} -eq 1 ]]; then printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" fi - if [[ ( -r test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig ) ]] ; then + if [[ ( -r test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig ) ]] ; then # configure your CI evironment to trust the key at gpg --keyserver keys.openpgp.org --recv-keys 9BD9E2DD46DA965A537E5B0A5CBF320243B6FD85 # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 # OR... @@ -185,10 +198,50 @@ for i in 1 256 512 ; do done if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - mv -f test-reporter-latest-${CI_OS:-linux}-amd64 ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 + mv -fv ./test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64} ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 chmod -v 751 ./cc-test-reporter || EXIT_CODE=77 fi +if [[ -z ${DEEPSOURCE_DSN} ]] && [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "%s\n" "No need for Unconfigured DeepSource." ; +elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "%s\n" "Detected Config for DeepSource" ; + + # USED FOR INTEGRETY + printf "adc12b89f5a41201f7556d4181cf33fa0a5a9964 ds-cli.sh\n" >ds-cli.sh.sha1 + printf "61e8a49d3d5c49a3614be2bcaaf96e232466fd87c225ecc782e96f4d65dd5bdf ds-cli.sh\n" >ds-cli.sh.sha256 + printf "b7232d1e8d73aeb4cefdf3707dd43f0e141b2585fedfd602cb97ec15047de60284f0ed4703b6e98717aebc3aee90e11c26417afbaba1168a5d0a98e22d74f66d ds-cli.sh\n" >ds-cli.sh.sha512 + + curl -fLso ./ds-cli.sh "https://deepsource.io/cli" || EXIT_CODE=125 ; + + for i in 1 256 512 ; do + # test sha1/sha512 signatures if found and sha256 even if not found + if [[ ( -r ds-cli.sh.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then + if [[ ${i} -eq 1 ]]; then + printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" + fi + shasum -a $i -c --ignore-missing ds-cli.sh.sha${i} || EXIT_CODE=126 + rm -vf ds-cli.sh.sha${i} 2>/dev/null ; + fi + done + + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + chmod -v 751 ./ds-cli.sh || EXIT_CODE=77 + chmod +x ./ds-cli.sh || : ; + fi ; + + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage + { sh ./ds-cli.sh ;} || EXIT_CODE=$? ; wait ; + fi ; + + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + # ref: https://docs.deepsource.com/docs/cli#commands + ./bin/deepsource config validate || EXIT_CODE=$? ; + fi ; + wait ; +fi ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then ./cc-test-reporter before-build || EXIT_CODE=125 ; fi @@ -200,3 +253,6 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait # goodbye exit ${EXIT_CODE:-255} ; + +# This file's code hash: +c1c7061d8f7686b221742d3b85c4a14db9699225f1766b6fbd68053642e1aea524736b93445554a1af14ed933ada4ed9 - From acf255fe7ec3b091d7fdcb1fe2df1a39e5545727 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Sun, 8 Dec 2024 21:37:20 -0800 Subject: [PATCH 04/58] [CI] Part 3 of many (- WIP PR #82 -) --- .github/workflows/Tests.yml | 4 ++-- tests/fetch_cc-test-reporter | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index ec7420b..dcfdfb1 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -213,10 +213,10 @@ jobs: id: clean run: make -j1 -f Makefile clean || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} - if: ${{ runner.python-version }} == "3.9" + if: ${{ runner.python-version == "3.9" }} run: make -f Makefile test ; - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} - if: ${{ runner.python-version }} != "3.9" + if: ${{ runner.python-version != "3.9" }} run: make -f Makefile test-pytest ; - name: Upload Python ${{ matrix.python-version }} coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index b9ba9af..a228ee1 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -70,7 +70,7 @@ diff -q <(tail -n 191 "$0" | head -n 189 | shasum -a 384 -t -) <(tail -n 1 "$0") ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" LANG=${LANG:-"en_US"} -LC_ALL="${LANG:1:5}.utf-8" +LC_ALL="${LANG:0:5}.utf-8" umask 127 LOCK_FILE="${TMPDIR:-/tmp}/org.pak.tests.scripts.code-climate.lock" @@ -255,4 +255,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -c1c7061d8f7686b221742d3b85c4a14db9699225f1766b6fbd68053642e1aea524736b93445554a1af14ed933ada4ed9 - +bd3a1cfef39983285334fad565c7a0643c208c3a7fb1a6aa4d19d44b537d21a5e2f8f13dbe3ed133e6f1435ed30c50ac - From b52b27208677a51e43ed857f2c4cde9f3bb09f84 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 17:18:40 -0700 Subject: [PATCH 05/58] [FIX] posible fix for #82 --- .github/workflows/Tests.yml | 12 ------------ Makefile | 3 ++- tests/fetch_cc-test-reporter | 5 ++--- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index dcfdfb1..4b3f482 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -58,18 +58,6 @@ jobs: lang-var: ["de.utf-8", "jp.utf-8"] experimental: [true] include: - - python-version: 3.7 - lang-var: "en_US.utf-8" - experimental: true - - python-version: 3.8 - lang-var: "en_US.utf-8" - experimental: true - - python-version: 3.9 - lang-var: "de.utf-8" - experimental: true - - python-version: 3.9 - lang-var: "jp.utf-8" - experimental: true - python-version: 3.9 lang-var: "en_US.utf-8" experimental: false diff --git a/Makefile b/Makefile index c400d8c..b85619f 100644 --- a/Makefile +++ b/Makefile @@ -222,6 +222,7 @@ test: just-test cc-test-reporter $(QUIET)$(DO_FAIL) ; $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)./cc-test-reporter after-build --exit-code 0 -t coverage.py 2>/dev/null || : ; $(QUIET)$(ECHO) "$@: Done." test-tox: cleanup @@ -231,7 +232,7 @@ test-tox: cleanup test-pytest: cleanup MANIFEST.in cc-test-reporter must_have_pytest test-reports $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=. || DO_FAIL="exit 2" ; $(QUIET)./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml || : ; - $(QUIET)./cc-test-reporter after-build --exit-code 0 -t coverage.py -r $(CC_TEST_REPORTER_ID) || : ; + $(QUIET)./cc-test-reporter after-build --exit-code 0 -t coverage.py 2>/dev/null || : ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index a228ee1..a01ddc9 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -158,7 +158,7 @@ if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); fi printf "::debug::%s\n" "Found ${_TEST_ROOT_DIR} ..." ; else - printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}::%s\n" \ + printf "::error %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}" \ "FAIL: missing valid repository or source structure" >&2 ; EXIT_CODE=40 fi @@ -227,7 +227,6 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then chmod -v 751 ./ds-cli.sh || EXIT_CODE=77 - chmod +x ./ds-cli.sh || : ; fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then @@ -255,4 +254,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -bd3a1cfef39983285334fad565c7a0643c208c3a7fb1a6aa4d19d44b537d21a5e2f8f13dbe3ed133e6f1435ed30c50ac - +162c5fd711e080b71f4943c910cec820eda71160c4a3c4af1d981104d5d2546e3d71a8b75a8ef1061d654e928becf0e1 - From 42ff73c48472b22ae20f40c99bbc79c30f1e4cb8 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 17:21:45 -0700 Subject: [PATCH 06/58] [FIX] resolved a config regression (- WIP PR #82 -) --- .flake8.ini | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.flake8.ini b/.flake8.ini index 6d4e752..b7b5d47 100644 --- a/.flake8.ini +++ b/.flake8.ini @@ -1,13 +1,14 @@ [flake8] select = C,E,F,W,B,B950 # Ignore specific warnings and errors according to CEP-8 style -extend-ignore = - W191, # Indentation contains tabs - W391, # Blank line at end of file - E117, # Over-indented - D208, # Docstring is over-indented - D203, # 1 blank line required before class docstring - CEP-7 - D212, # Multi-line docstring summary should start at the first line - CEP-7 +extend-ignore = E117,D203,D208,D212,W191,W391 +# CEP-8 Custom Exceptions: +# W191, # Indentation contains tabs +# W391, # Blank line at end of file +# E117, # Over-indented +# D208, # Docstring is over-indented +# D203, # 1 blank line required before class docstring - CEP-7 +# D212, # Multi-line docstring summary should start at the first line - CEP-7 # Ignore long lines as specified in CEP-8 max-line-length = 100 extend-exclude = From b4b8cd1c468f9183dd54af51a791f7d80110ffdf Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 17:37:35 -0700 Subject: [PATCH 07/58] [FIX] more fixes from review (- WIP PR #82 -) --- .flake8.ini | 2 +- pyproject.tomal => pyproject.toml | 0 requirements.txt | 17 ++++++++++------- setup.cfg | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) rename pyproject.tomal => pyproject.toml (100%) diff --git a/.flake8.ini b/.flake8.ini index b7b5d47..5337f6c 100644 --- a/.flake8.ini +++ b/.flake8.ini @@ -1,5 +1,5 @@ [flake8] -select = C,E,F,W,B,B950 +select = C,D,E,F,W,B,B950 # Ignore specific warnings and errors according to CEP-8 style extend-ignore = E117,D203,D208,D212,W191,W391 # CEP-8 Custom Exceptions: diff --git a/pyproject.tomal b/pyproject.toml similarity index 100% rename from pyproject.tomal rename to pyproject.toml diff --git a/requirements.txt b/requirements.txt index 6841f4b..2757206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,18 +4,21 @@ # subprocess - PSF licence # sphinx # argparse - builtin - PSF licence -argparse>=1.4.0 -# argparse - builtin - PSF licence -setuptools>=38.0 -# virtualenv - MIT -virtualenv>=15.0.1 # six - MIT six>=1.0.0 # pgpy - BSD 3-Clause licensed #pgpy>=0.4.1 tox>=3.0.0 #py>=1.4.33 -pip>=19.0 +# setuptools - MIT license +setuptools>=75.0 +# virtualenv - MIT license +virtualenv>=20.26.6 +# pip - MIT license +pip>=24.3.1 +# build - MIT license +build>=1.1.1, !=1.2.2.post1 # python-repo - MIT #-e git+https://github.com/reactive-firewall/python-repo.git#egg=python-repo - +# wheel - MIT license +wheel>=0.44 diff --git a/setup.cfg b/setup.cfg index 603be15..209f6d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,9 +72,9 @@ testing = pytest-enabler >= 1.0.1 # local flake8 >= 5.0 - virtualenv >= 20.26.5 + virtualenv >= 20.26.6 wheel >= 0.44.0 - pip >= 22.0 + pip >= 24.3.1 pytest-cov >= 4.0.0; \ # coverage seems to make PyPy extremely slow python_implementation != "PyPy" From b704d72d8f41a3bcc3d82fc75f029ae20e3f4429 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 17:50:01 -0700 Subject: [PATCH 08/58] [FIX] more fixes for circle-CI this time. --- .circleci/config.yml | 127 +++++++++++++++++++------------------------ 1 file changed, 55 insertions(+), 72 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e22ae8..345cc0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,18 +1,30 @@ -version: 2 +--- +version: 2.1 +commands: + cleanup: + steps: + - run: + shell: /bin/bash + name: "Cleanup" + command: | + make clean + when: always + +parameters: + python-version: + type: string + default: "3.12" + jobs: build: docker: - - image: cimg/python:3.7 - - image: cimg/python:3.8 - - image: cimg/python:3.9 - - image: cimg/python:3.10 - - image: cimg/python:3.11 - - image: cimg/python:3.12 + - image: cimg/python:<< pipeline.parameters.python-version >> + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 SHELL: /bin/bash working_directory: ~/python-repo steps: @@ -20,18 +32,19 @@ jobs: - run: name: "fetch and pull" command: | - git fetch && git pull --all || true + git fetch && git pull --all || : ; - run: shell: /bin/bash name: "install depends attempt" command: | - python3 -m pip install --user -r ./requirements.txt || true + python3 -m pip install --user -r ./requirements.txt || : ; when: on_success - run: shell: /bin/bash - name: "install test extras attempt" + name: "install test depends attempt" command: | - python3 -m pip install --user -r ./test-requirements.txt || true + python3 -m pip install --upgrade --user -r ./tests/requirements.txt || : ; + when: on_success - save_cache: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} paths: @@ -39,18 +52,14 @@ jobs: test: docker: - - image: cimg/python:3.7 - - image: cimg/python:3.8 - - image: cimg/python:3.9 - - image: cimg/python:3.10 - - image: cimg/python:3.11 - - image: cimg/python:3.12 + - image: cimg/python:<< pipeline.parameters.python-version >> parallelism: 2 + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 SHELL: /bin/bash working_directory: ~/python-repo steps: @@ -58,43 +67,29 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - run: shell: /bin/bash - name: "clean up for test" + name: "Installing deps for test" command: | - make clean - when: always + python3 -m pip install --upgrade --user -r ./tests/requirements.txt || : ; + when: on_success + - cleanup - run: shell: /bin/bash name: "Unit Tests" command: | make test when: on_success - - run: - shell: /bin/bash - name: "clean up from test" - command: | - make clean - when: always - - run: - shell: /bin/bash - name: "clean up from FAIL" - command: | - make clean - when: on_fail + - cleanup pytest: docker: - - image: cimg/python:3.7 - - image: cimg/python:3.8 - - image: cimg/python:3.9 - - image: cimg/python:3.10 - - image: cimg/python:3.11 - - image: cimg/python:3.12 + - image: cimg/python:<< pipeline.parameters.python-version >> parallelism: 2 + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 SHELL: /bin/bash working_directory: ~/python-repo steps: @@ -102,16 +97,17 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - run: shell: /bin/bash - name: "setup up for pytest" + name: "set up depends" command: | - python3 -m pip install --upgrade --user -r ./test-requirements.txt || true + python3 -m pip install --upgrade --user -r ./requirements.txt || : ; when: on_success - run: shell: /bin/bash - name: "clean up for pytest" + name: "clean up for test" command: | - make clean - when: always + python3 -m pip install --upgrade --user -r ./tests/requirements.txt || : ; + when: on_success + - cleanup - run: shell: /bin/bash name: "pytest Unit Tests" @@ -124,55 +120,41 @@ jobs: - store_artifacts: path: test-reports when: on_success - - run: - shell: /bin/bash - name: "clean up from pytest" - command: | - make clean - when: always - - run: - shell: /bin/bash - name: "clean up from FAIL" - command: | - make clean - when: on_fail + - cleanup lint: docker: - - image: cimg/python:3.11 + - image: cimg/python:<< pipeline.parameters.python-version >> + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 SHELL: /bin/bash - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 working_directory: ~/python-repo steps: - restore_cache: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - run: - name: "install linters extras attempt" + name: "install reqs attempt" command: | - python3 -m pip install --upgrade --user -r ./test-requirements.txt || true + python3 -m pip install --user -r ./requirements.txt || : ; - run: - shell: /bin/bash - name: "clean up for test" + name: "install test-reqs attempt" command: | - make clean + python3 -m pip install --user -r ./tests/requirements.txt || : ; + - cleanup - run: shell: /bin/bash name: "check code style and spelling" command: | - make test-style || python3 -m flake8 --ignore=W191,W391,E117 --max-line-length=100 --verbose --count --config=.flake8.ini --max-complexity=10 - - run: - shell: /bin/bash - name: "clean up when done" - command: | - make clean + make test-style || python3 -m flake8 --verbose --count --config=.flake8.ini + - cleanup workflows: version: 2 - workflow: + test-matrix: jobs: - build - test: @@ -184,3 +166,4 @@ workflows: - pytest: requires: - build + - test From 5ae38a25338c90a8c6b0fbec98299563f9054ada Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 18:05:34 -0700 Subject: [PATCH 09/58] [FIX] fix for Circle CI edge case --- tests/fetch_cc-test-reporter | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index a01ddc9..ca038cd 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -189,7 +189,7 @@ for i in 1 256 512 ; do # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 # OR... # Set CI=true to continue on missing keys - gpg --verify test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || ${CI} || EXIT_CODE=126 + gpg --verify test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=126 rm -vf test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig 2>/dev/null ; fi shasum -a $i -c --ignore-missing test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || EXIT_CODE=126 @@ -254,4 +254,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -162c5fd711e080b71f4943c910cec820eda71160c4a3c4af1d981104d5d2546e3d71a8b75a8ef1061d654e928becf0e1 - +162b42710735f3bad7c4912f5053d77ad52809bb53ad80bf250bf0b1af4dce6df2d12744328159ce06e2ce00a4c21e6bee8 - From 6a11f9acb1a0deb2228ed47017279c85548e9e45 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 18:09:05 -0700 Subject: [PATCH 10/58] [FIX] Regression fix for last commit --- tests/fetch_cc-test-reporter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index ca038cd..8bb1376 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -254,4 +254,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -162b42710735f3bad7c4912f5053d77ad52809bb53ad80bf250bf0b1af4dce6df2d12744328159ce06e2ce00a4c21e6bee8 - +b42710735f3bad7c4912f5053d77ad52809bb53ad80bf250bf0b1af4dce6df2d12744328159ce06e2ce00a4c21e6bee8 - From a600f380782b74c9d06d96a6fd9c8c9bfc9f424d Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 18:15:21 -0700 Subject: [PATCH 11/58] [FIX] fix for regression typo in config for CI. --- .github/workflows/Tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 4b3f482..4377453 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -201,10 +201,10 @@ jobs: id: clean run: make -j1 -f Makefile clean || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} - if: ${{ runner.python-version == "3.9" }} + if: ${{ runner.python-version == '3.9' }} run: make -f Makefile test ; - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} - if: ${{ runner.python-version != "3.9" }} + if: ${{ runner.python-version != '3.9' }} run: make -f Makefile test-pytest ; - name: Upload Python ${{ matrix.python-version }} coverage to Codecov uses: codecov/codecov-action@v5 From d3efa637a0fd120714079d5f06ad47a1ace96813 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 19:28:32 -0700 Subject: [PATCH 12/58] [DEBUG] added more debug code to tool (- WIP PR #82 -) --- tests/fetch_cc-test-reporter | 40 ++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 8bb1376..e39bfa8 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -65,7 +65,7 @@ # to check though: -diff -q <(tail -n 191 "$0" | head -n 189 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; +diff -q <(tail -n 228 "$0" | head -n 226 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" @@ -214,6 +214,10 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then curl -fLso ./ds-cli.sh "https://deepsource.io/cli" || EXIT_CODE=125 ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::debug::%s\n" "Downloaded ds-cli.sh for DeepSource" ; + fi ; + for i in 1 256 512 ; do # test sha1/sha512 signatures if found and sha256 even if not found if [[ ( -r ds-cli.sh.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then @@ -226,12 +230,44 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then done if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::debug::%s\n" "Validated ds-cli.sh for DeepSource" ; chmod -v 751 ./ds-cli.sh || EXIT_CODE=77 fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage + printf "::group::%s\n" "Running ./ds-cli.sh" ; { sh ./ds-cli.sh ;} || EXIT_CODE=$? ; wait ; + printf "::endgroup::\n" ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::debug::%s\n" "Checking for DeepSource CLI" ; + if [[ ( -d ./bin ) ]] ; then + printf "::debug::==> %s\n" "Found ./bin directory." ; + if [[ ( -x ./bin/deepsource ) ]] ; then + printf "::debug:: ==> %s\n" "Found ./bin/deepsource CLI tool" ; + elif [[ ( -r ./bin/deepsource ) || ( -e ./bin/deepsource ) ]] ; then + printf "::debug:: ==> %s\n" "Found ./bin/deepsource file. Permission Error" ; + printf "::debug:: ==> %s\n" "Trying to apply permission corrections" ; + chmod -v 751 ./bin/deepsource || EXIT_CODE=77 ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] && [[ ( -x ./bin/deepsource ) ]] ; then + printf "::debug:: ==> %s\n" "Fixed ./bin/deepsource CLI tool" ; + else + printf "::debug:: ==> %s\n" "Applying corrections Unsuccessful" ; + EXIT_CODE=77 ; + if + else + printf "::debug:: ==> %s\n" "Missing ./bin/deepsource CLI tool" ; + EXIT_CODE=125 ; + printf "::debug:: ==> %s\n" "Checking if bin is empty:" ; + readonly BIN_FILES=$( ls -1 ./bin 2>/dev/null ;) + printf "::debug:: ==> %s\n" "${BIN_FILES}" ; + unset $BIN_FILES 2>/dev/null || : ; + fi ; + else + printf "::debug::==> %s\n" "Missing ./bin directory." ; + EXIT_CODE=125 ; + fi ; + fi ; fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then @@ -254,4 +290,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -b42710735f3bad7c4912f5053d77ad52809bb53ad80bf250bf0b1af4dce6df2d12744328159ce06e2ce00a4c21e6bee8 - +a3ffe4db431f0e6b2d32e6f4969a848da9b14257ae3461e968352825ab50ae7db48fd20359378c49d7f7e3ec2f3402d3 - From 2688da8fd9bd49776e79a42b48a5ae6133ca705f Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 19:32:51 -0700 Subject: [PATCH 13/58] [DEBUG] fix regression from last commit (- WIP #82 -) --- tests/fetch_cc-test-reporter | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index e39bfa8..226283d 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -254,7 +254,7 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then else printf "::debug:: ==> %s\n" "Applying corrections Unsuccessful" ; EXIT_CODE=77 ; - if + fi ; else printf "::debug:: ==> %s\n" "Missing ./bin/deepsource CLI tool" ; EXIT_CODE=125 ; @@ -290,4 +290,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -a3ffe4db431f0e6b2d32e6f4969a848da9b14257ae3461e968352825ab50ae7db48fd20359378c49d7f7e3ec2f3402d3 - +99229c350d321d5557c1833263cdbe4b09271ad374d4f81eb08d19d8ae00c922e3764eaf7ccfb506f6e58171c2ecf5f6 - From 97df053e21802a7d9165ed7161e85eebc1ba2705 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 20:29:49 -0700 Subject: [PATCH 14/58] [DEBUG] more debugging --- .circleci/config.yml | 8 ++--- .github/workflows/Tests.yml | 6 ++-- test-requirements.txt | 21 +++++++----- tests/fetch_cc-test-reporter | 64 ++++++++++++++++++++---------------- 4 files changed, 55 insertions(+), 44 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 345cc0a..a8be83f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: shell: /bin/bash name: "install test depends attempt" command: | - python3 -m pip install --upgrade --user -r ./tests/requirements.txt || : ; + python3 -m pip install --upgrade --user -r ./tests-requirements.txt || : ; when: on_success - save_cache: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} @@ -69,7 +69,7 @@ jobs: shell: /bin/bash name: "Installing deps for test" command: | - python3 -m pip install --upgrade --user -r ./tests/requirements.txt || : ; + python3 -m pip install --upgrade --user -r ./tests-requirements.txt || : ; when: on_success - cleanup - run: @@ -105,7 +105,7 @@ jobs: shell: /bin/bash name: "clean up for test" command: | - python3 -m pip install --upgrade --user -r ./tests/requirements.txt || : ; + python3 -m pip install --upgrade --user -r ./tests-requirements.txt || : ; when: on_success - cleanup - run: @@ -143,7 +143,7 @@ jobs: - run: name: "install test-reqs attempt" command: | - python3 -m pip install --user -r ./tests/requirements.txt || : ; + python3 -m pip install --user -r ./tests-requirements.txt || : ; - cleanup - run: shell: /bin/bash diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 4377453..2dd85c4 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -193,13 +193,13 @@ jobs: pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; pip install -r ./requirements.txt ; pip install -r ./test-requirements.txt || true ; + - name: Pre-Clean + id: clean + run: make -j1 -f Makefile clean || true ; - name: Install code-climate tools for ${{ matrix.python-version }} if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash run: ./tests/fetch_cc-test-reporter || true ; - - name: Pre-Clean - id: clean - run: make -j1 -f Makefile clean || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} if: ${{ runner.python-version == '3.9' }} run: make -f Makefile test ; diff --git a/test-requirements.txt b/test-requirements.txt index b09086e..6ede96c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,19 +3,24 @@ # re - builtin - PSF licence? # subprocess - PSF licence # sphinx -# argparse - builtin - PSF licence -argparse>=1.4.0 -# argparse - builtin - PSF licence -setuptools>=38.0 -# virtualenv - MIT -virtualenv>=15.0.1 # six - MIT six>=1.0.0 # pgpy - BSD 3-Clause licensed #pgpy>=0.4.1 tox>=3.0.0 #py>=1.4.33 -# pip>=19.0 +# setuptools - MIT license +setuptools>=75.0 +# virtualenv - MIT license +virtualenv>=20.26.6 +# pip - MIT license +pip>=24.3.1 +# build - MIT license +build>=1.1.1, !=1.2.2.post1 +# python-repo - MIT +#-e git+https://github.com/reactive-firewall/python-repo.git#egg=python-repo +# wheel - MIT license +wheel>=0.44 # # TESTING ONLY - Do NOT report issues with these optionals on python-repo # @@ -30,5 +35,3 @@ pytest-flake8>=1.0 coverage >= 6.3 pytest-cov >= 4.0.0; pytest-enabler >= 1.0.1 -wheel >= 0.37.0 -pip >= 21.0 diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 226283d..0162666 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -65,12 +65,13 @@ # to check though: -diff -q <(tail -n 228 "$0" | head -n 226 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; +diff -q <(tail -n 229 "$0" | head -n 227 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" LANG=${LANG:-"en_US"} LC_ALL="${LANG:0:5}.utf-8" +export BINDIR="${BINDIR:-./bin}" umask 127 LOCK_FILE="${TMPDIR:-/tmp}/org.pak.tests.scripts.code-climate.lock" @@ -118,7 +119,14 @@ case "$ARCH" in esac # Set the CI prefix -CI_ENV_PREFIX="${CI_OS}-${ARCH}" +case "${CI_OS:-Linux}" in + *arwin) + CI_ENV_PREFIX="${CI_OS}-amd64" + ;; + *) + CI_ENV_PREFIX="${CI_OS}-${ARCH}" + ;; +esac function cleanup() { rm -f "${LOCK_FILE}" 2>/dev/null || : ; wait ; @@ -175,12 +183,12 @@ fi ######################### # actual Work starts here ######################### -curl -fLso ./test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64} https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64} ; +curl -fLso ./test-reporter-latest-${CI_ENV_PREFIX} https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX} ; for i in 1 256 512 ; do - curl -fLso test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}" ; wait ; - curl -fLso test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig" ; wait ; + curl -fLso test-reporter-latest-${CI_ENV_PREFIX}.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}.sha${i}" ; wait ; + curl -fLso test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig" ; wait ; # test sha1/sha512 signatures if found and sha256 even if not found - if [[ ( -r test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then + if [[ ( -r test-reporter-latest-${CI_ENV_PREFIX}.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then if [[ ${i} -eq 1 ]]; then printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" fi @@ -189,16 +197,16 @@ for i in 1 256 512 ; do # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 # OR... # Set CI=true to continue on missing keys - gpg --verify test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig 2>/dev/null ; + gpg --verify test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=126 + rm -vf test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig 2>/dev/null ; fi - shasum -a $i -c --ignore-missing test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i} 2>/dev/null ; + shasum -a $i -c --ignore-missing test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || EXIT_CODE=126 + rm -vf test-reporter-latest-${CI_ENV_PREFIX}.sha${i} 2>/dev/null ; fi done if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - mv -fv ./test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64} ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 + mv -fv ./test-reporter-latest-${CI_ENV_PREFIX} ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 chmod -v 751 ./cc-test-reporter || EXIT_CODE=77 fi @@ -241,30 +249,30 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::endgroup::\n" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Checking for DeepSource CLI" ; - if [[ ( -d ./bin ) ]] ; then - printf "::debug::==> %s\n" "Found ./bin directory." ; - if [[ ( -x ./bin/deepsource ) ]] ; then - printf "::debug:: ==> %s\n" "Found ./bin/deepsource CLI tool" ; - elif [[ ( -r ./bin/deepsource ) || ( -e ./bin/deepsource ) ]] ; then - printf "::debug:: ==> %s\n" "Found ./bin/deepsource file. Permission Error" ; - printf "::debug:: ==> %s\n" "Trying to apply permission corrections" ; - chmod -v 751 ./bin/deepsource || EXIT_CODE=77 ; - if [[ ( ${EXIT_CODE} -eq 0 ) ]] && [[ ( -x ./bin/deepsource ) ]] ; then - printf "::debug:: ==> %s\n" "Fixed ./bin/deepsource CLI tool" ; + if [[ ( -d "${BINDIR}" ) ]] ; then + printf "::debug::=> %s\n" "Found ${BINDIR} directory." ; + if [[ ( -x "${BINDIR}"/deepsource ) ]] ; then + printf "::debug::..=> %s\n" "Found ${BINDIR}/deepsource CLI tool" ; + elif [[ ( -r "${BINDIR}"/deepsource ) || ( -e "${BINDIR}"/deepsource ) ]] ; then + printf "::debug::..=> %s\n" "Found ${BINDIR}/deepsource file." ; + printf "::debug::....=> %s\n" "Trying to apply permission corrections" ; + chmod -v 751 "${BINDIR}"/deepsource || EXIT_CODE=77 ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] && [[ ( -x "${BINDIR}"/deepsource ) ]] ; then + printf "::debug::......=> %s\n" "Fixed ${BINDIR}/deepsource CLI tool" ; else - printf "::debug:: ==> %s\n" "Applying corrections Unsuccessful" ; + printf "::debug::....=> %s\n" "Applying corrections Unsuccessful" ; EXIT_CODE=77 ; fi ; else - printf "::debug:: ==> %s\n" "Missing ./bin/deepsource CLI tool" ; + printf "::debug::..=> %s\n" "Missing ${BINDIR}/deepsource CLI tool" ; EXIT_CODE=125 ; - printf "::debug:: ==> %s\n" "Checking if bin is empty:" ; - readonly BIN_FILES=$( ls -1 ./bin 2>/dev/null ;) - printf "::debug:: ==> %s\n" "${BIN_FILES}" ; + printf "::debug::..=> %s\n" "Checking if ${BINDIR} is empty:" ; + readonly BIN_FILES=$( ls -1 "${BINDIR}" 2>/dev/null ;) + printf "::debug::..=> %s\n" "${BIN_FILES}" ; unset $BIN_FILES 2>/dev/null || : ; fi ; else - printf "::debug::==> %s\n" "Missing ./bin directory." ; + printf "::debug::=> %s\n" "Missing ${BINDIR} directory." ; EXIT_CODE=125 ; fi ; fi ; @@ -290,4 +298,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -99229c350d321d5557c1833263cdbe4b09271ad374d4f81eb08d19d8ae00c922e3764eaf7ccfb506f6e58171c2ecf5f6 - +c107fcd6c083b3fcfe410ce7fa6017a27c519172511d959f52ded59cd8af8e8648dcd58ff27d624df32284d1a9a4289a - From 09731abe2fbb51edf48e36ba799f0597623ba0ad Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 20:34:29 -0700 Subject: [PATCH 15/58] [DEBUG] More clean up and tweaks. Does it work yet? --- tests/fetch_cc-test-reporter | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 0162666..17d3006 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -65,7 +65,7 @@ # to check though: -diff -q <(tail -n 229 "$0" | head -n 227 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; +diff -q <(tail -n 234 "$0" | head -n 232 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" @@ -298,4 +298,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -c107fcd6c083b3fcfe410ce7fa6017a27c519172511d959f52ded59cd8af8e8648dcd58ff27d624df32284d1a9a4289a - +f655d5970127b38f5066a87c0ecbc9f413fe899816f7695b24b99eedae25e5c44ee278e9d394613db7d4d4b1b6a567c6 - From f6188e5075667e7e5039956c6768000908ea3e23 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 21:00:49 -0700 Subject: [PATCH 16/58] [DEBUG] Even more fixes. Does it work now? --- .circleci/config.yml | 8 ++++---- Makefile | 27 +++++++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a8be83f..a1bf887 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: shell: /bin/bash name: "install test depends attempt" command: | - python3 -m pip install --upgrade --user -r ./tests-requirements.txt || : ; + python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; when: on_success - save_cache: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} @@ -69,7 +69,7 @@ jobs: shell: /bin/bash name: "Installing deps for test" command: | - python3 -m pip install --upgrade --user -r ./tests-requirements.txt || : ; + python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; when: on_success - cleanup - run: @@ -105,7 +105,7 @@ jobs: shell: /bin/bash name: "clean up for test" command: | - python3 -m pip install --upgrade --user -r ./tests-requirements.txt || : ; + python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; when: on_success - cleanup - run: @@ -143,7 +143,7 @@ jobs: - run: name: "install test-reqs attempt" command: | - python3 -m pip install --user -r ./tests-requirements.txt || : ; + python3 -m pip install --user -r ./test-requirements.txt || : ; - cleanup - run: shell: /bin/bash diff --git a/Makefile b/Makefile index b85619f..d9dbf0b 100644 --- a/Makefile +++ b/Makefile @@ -91,16 +91,31 @@ else endif endif +ifndef CC_TOOL + FETCH_CC_TOOL := tests/fetch_cc-test-reporter + CC_TOOL := ./cc-test-reporter + CC_TOOL_ARGS := after-build --exit-code 0 -t coverage.py + DS_TOOL := ./bin/deepsource + DS_TOOL_ARGS := report --analyzer test-coverage --key python --value-file ./coverage.xml +endif + ifndef PIP_COMMON_FLAGS # Define common pip install flags PIP_COMMON_FLAGS := --use-pep517 --exists-action s --upgrade --upgrade-strategy eager endif -# Define environment-specific pip install flags -ifeq ($(shell uname),Darwin) +# Define environment-specific flags +ifeq ($(shell uname -s), Darwin) PIP_ENV_FLAGS := --break-system-packages +else ifeq ($(shell uname -s), Linux) + PIP_ENV_FLAGS := else PIP_ENV_FLAGS := + FETCH_CC_TOOL := + CC_TOOL := + CC_TOOL_ARGS := + DS_TOOL := + DS_TOOL_ARGS := endif ifeq "$(WAIT)" "" @@ -222,7 +237,7 @@ test: just-test cc-test-reporter $(QUIET)$(DO_FAIL) ; $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; - $(QUIET)./cc-test-reporter after-build --exit-code 0 -t coverage.py 2>/dev/null || : ; + $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(ECHO) "$@: Done." test-tox: cleanup @@ -231,8 +246,8 @@ test-tox: cleanup test-pytest: cleanup MANIFEST.in cc-test-reporter must_have_pytest test-reports $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=. || DO_FAIL="exit 2" ; - $(QUIET)./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml || : ; - $(QUIET)./cc-test-reporter after-build --exit-code 0 -t coverage.py 2>/dev/null || : ; + $(QUIET)$(DS_TOOL) $(DS_TOOL_ARGS) || : ; + $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." @@ -244,7 +259,7 @@ test-style: cleanup $(QUIET)$(ECHO) "$@: Done." cc-test-reporter: tests/fetch_cc-test-reporter - $(QUIET)tests/fetch_cc-test-reporter ; + $(QUIET)$(FETCH_CC_TOOL) ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." From 286b0793d3589cdc61d55ee887905a87dfc61af6 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 21:19:56 -0700 Subject: [PATCH 17/58] [DEBUG] more tricks to fool deepsource hacks. does it work now? --- tests/fetch_cc-test-reporter | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 17d3006..d12c2b5 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -245,7 +245,7 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage printf "::group::%s\n" "Running ./ds-cli.sh" ; - { sh ./ds-cli.sh ;} || EXIT_CODE=$? ; wait ; + time { cat ./ds-cli.sh | sh ;} || EXIT_CODE=$? ; wait ; printf "::endgroup::\n" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Checking for DeepSource CLI" ; @@ -298,4 +298,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -f655d5970127b38f5066a87c0ecbc9f413fe899816f7695b24b99eedae25e5c44ee278e9d394613db7d4d4b1b6a567c6 - +c7f8f4f1cf337ce97fe32916af5507aa252f7ec441eb6105fcf3fe32d5463994e8889539cff613f5dfff802324fb0bc0 - From 4febba33acb944545199bad35e5b7b0813bc5ea1 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 22:13:00 -0700 Subject: [PATCH 18/58] [DEBUG] break a few eggs for the omlet --- .github/workflows/Tests.yml | 5 ++++- Makefile | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 2dd85c4..14607ac 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -199,7 +199,10 @@ jobs: - name: Install code-climate tools for ${{ matrix.python-version }} if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash - run: ./tests/fetch_cc-test-reporter || true ; + run: | + ./tests/fetch_cc-test-reporter || true ; + ls -lap . || true ; + ls -lap bin || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} if: ${{ runner.python-version == '3.9' }} run: make -f Makefile test ; diff --git a/Makefile b/Makefile index d9dbf0b..f2c4b40 100644 --- a/Makefile +++ b/Makefile @@ -111,10 +111,10 @@ else ifeq ($(shell uname -s), Linux) PIP_ENV_FLAGS := else PIP_ENV_FLAGS := - FETCH_CC_TOOL := - CC_TOOL := + FETCH_CC_TOOL := : + CC_TOOL := : CC_TOOL_ARGS := - DS_TOOL := + DS_TOOL := : DS_TOOL_ARGS := endif @@ -259,7 +259,7 @@ test-style: cleanup $(QUIET)$(ECHO) "$@: Done." cc-test-reporter: tests/fetch_cc-test-reporter - $(QUIET)$(FETCH_CC_TOOL) ; + $(QUIET)$(FETCH_CC_TOOL) || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." From 58f427c504866799c62ecc84346a75509a08a1e6 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 22:24:10 -0700 Subject: [PATCH 19/58] [DEBUG] More fixes. Does it work yet? --- .github/workflows/Tests.yml | 5 +---- tests/fetch_cc-test-reporter | 12 +++++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 14607ac..2dd85c4 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -199,10 +199,7 @@ jobs: - name: Install code-climate tools for ${{ matrix.python-version }} if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash - run: | - ./tests/fetch_cc-test-reporter || true ; - ls -lap . || true ; - ls -lap bin || true ; + run: ./tests/fetch_cc-test-reporter || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} if: ${{ runner.python-version == '3.9' }} run: make -f Makefile test ; diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index d12c2b5..46556d4 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -65,7 +65,7 @@ # to check though: -diff -q <(tail -n 234 "$0" | head -n 232 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; +diff -q <(tail -n 240 "$0" | head -n 238 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" @@ -79,6 +79,7 @@ EXIT_CODE=0 test -x "$(command -v grep)" || exit 126 ; test -x "$(command -v curl)" || exit 126 ; +test -x "$(command -v mkdir)" || exit 126 ; hash -p ./.github/tool_shlock_helper.sh shlock || exit 255 ; test -x "$(command -v shlock)" || exit 126 ; test -x "$(command -v gpgv)" || exit 126 ; @@ -241,6 +242,11 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Validated ds-cli.sh for DeepSource" ; chmod -v 751 ./ds-cli.sh || EXIT_CODE=77 fi ; + + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::debug::%s\n" "Creating ${BINDIR} directory for DeepSource" ; + mkdir -v -m 755 "${BINDIR}" || EXIT_CODE=66 + fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage @@ -265,7 +271,7 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then fi ; else printf "::debug::..=> %s\n" "Missing ${BINDIR}/deepsource CLI tool" ; - EXIT_CODE=125 ; + EXIT_CODE=66 ; printf "::debug::..=> %s\n" "Checking if ${BINDIR} is empty:" ; readonly BIN_FILES=$( ls -1 "${BINDIR}" 2>/dev/null ;) printf "::debug::..=> %s\n" "${BIN_FILES}" ; @@ -298,4 +304,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -c7f8f4f1cf337ce97fe32916af5507aa252f7ec441eb6105fcf3fe32d5463994e8889539cff613f5dfff802324fb0bc0 - +bee43a56d600679736436b75cc9957dd2b6c14045498772a4c5e919449a74088a48c155ff70072be0ae5a268aa854bfe - From 90d9316388f752f6f7502d58258a2406b526dcb3 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 22:30:19 -0700 Subject: [PATCH 20/58] [DEBUG] WTF is going on with deepsource. it is still broken here. --- tests/fetch_cc-test-reporter | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 46556d4..bb8b712 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -251,7 +251,7 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage printf "::group::%s\n" "Running ./ds-cli.sh" ; - time { cat ./ds-cli.sh | sh ;} || EXIT_CODE=$? ; wait ; + time { cat ./ds-cli.sh | sh ;} ; wait ; printf "::endgroup::\n" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Checking for DeepSource CLI" ; @@ -304,4 +304,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -bee43a56d600679736436b75cc9957dd2b6c14045498772a4c5e919449a74088a48c155ff70072be0ae5a268aa854bfe - +b444b5d3e9093ee01e78a8fe991e60d0f6d48218fbb13cf8d3e6d06705a4289a7def2fcfe44758328c76572633cfd4ad - From 5040345d99cdb74ee234c5c9427ecb4a8b03040d Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 19 Mar 2025 22:43:15 -0700 Subject: [PATCH 21/58] [DEBUG] Stable state. but it does not work yet! --- .hound.yml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .hound.yml diff --git a/.hound.yml b/.hound.yml deleted file mode 100644 index 4f2d5e6..0000000 --- a/.hound.yml +++ /dev/null @@ -1,5 +0,0 @@ -python: - enabled: true - config_file: .flake8.ini - -fail_on_violations: true From 297e7586480f2753af058fea84cf475b8143e70d Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 20 Mar 2025 21:14:46 -0700 Subject: [PATCH 22/58] [DEBUG] Re-write tool to actually work (- WIP PR #82 -) --- Makefile | 2 + tests/fetch_cc-test-reporter | 81 ++++++++++++++++++++++++------------ 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index f2c4b40..ba82364 100644 --- a/Makefile +++ b/Makefile @@ -216,6 +216,8 @@ legacy-purge: clean uninstall purge: legacy-purge $(QUIET)$(RM) ./cc-test-reporter 2>$(ERROR_LOG_PATH) || : $(QUIET)$(RM) ./ds-cli.sh 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./bin/deepsource 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./bin/ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(RM) ./test-reports/*.xml 2>$(ERROR_LOG_PATH) || : $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index bb8b712..3c4b6db 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -65,25 +65,28 @@ # to check though: -diff -q <(tail -n 240 "$0" | head -n 238 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; +diff -q <(tail -n 269 "$0" | head -n 267 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" LANG=${LANG:-"en_US"} LC_ALL="${LANG:0:5}.utf-8" -export BINDIR="${BINDIR:-./bin}" +BINDIR="${BINDIR:-./bin}" +DS_TAG="0.8.6" # circa 2025 umask 127 LOCK_FILE="${TMPDIR:-/tmp}/org.pak.tests.scripts.code-climate.lock" EXIT_CODE=0 test -x "$(command -v grep)" || exit 126 ; +test -x "$(command -v sed)" || exit 126 ; test -x "$(command -v curl)" || exit 126 ; test -x "$(command -v mkdir)" || exit 126 ; hash -p ./.github/tool_shlock_helper.sh shlock || exit 255 ; test -x "$(command -v shlock)" || exit 126 ; test -x "$(command -v gpgv)" || exit 126 ; test -x "$(command -v shasum)" || exit 126 ; +test -x "$(command -v tar)" || exit 126 ; # Detect the operating system case "$( command uname -s )" in @@ -216,43 +219,69 @@ if [[ -z ${DEEPSOURCE_DSN} ]] && [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "%s\n" "Detected Config for DeepSource" ; - # USED FOR INTEGRETY - printf "adc12b89f5a41201f7556d4181cf33fa0a5a9964 ds-cli.sh\n" >ds-cli.sh.sha1 - printf "61e8a49d3d5c49a3614be2bcaaf96e232466fd87c225ecc782e96f4d65dd5bdf ds-cli.sh\n" >ds-cli.sh.sha256 - printf "b7232d1e8d73aeb4cefdf3707dd43f0e141b2585fedfd602cb97ec15047de60284f0ed4703b6e98717aebc3aee90e11c26417afbaba1168a5d0a98e22d74f66d ds-cli.sh\n" >ds-cli.sh.sha512 + # Make an API call to GitHub to get the latest release with the Accept header + response=$(curl -s -H "Accept: application/json" "https://api.github.com/repos/DeepSourceCorp/cli/releases/latest") - curl -fLso ./ds-cli.sh "https://deepsource.io/cli" || EXIT_CODE=125 ; + # Check if the response is valid + if [[ "$response" == *"Not Found"* ]]; then + printf "::debug::%s\n" "Repository not found or no releases available." + EXIT_CODE=125 + fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Downloaded ds-cli.sh for DeepSource" ; + printf "::debug::%s\n" "Checking latest version for DeepSource" ; + # Try to extract the tag name using jq + if [[ ( -x "$(command -v jq)" ) ]]; then + tag_name=$(printf "%s\n" "$response" | jq -r '.tag_name') + else + # Fallback to grep and sed if jq is not available + tag_name=$(printf "%s\n" "$response" | grep -o '"tag_name": "[^"]*' | sed 's/"tag_name": "//') + fi ; fi ; - for i in 1 256 512 ; do - # test sha1/sha512 signatures if found and sha256 even if not found - if [[ ( -r ds-cli.sh.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then - if [[ ${i} -eq 1 ]]; then - printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" - fi - shasum -a $i -c --ignore-missing ds-cli.sh.sha${i} || EXIT_CODE=126 - rm -vf ds-cli.sh.sha${i} 2>/dev/null ; - fi - done + # Check if tag_name is empty + if [[ ( -z "$tag_name" ) ]]; then + printf "::debug::%s\n" "No tag name found in the response." + EXIT_CODE=125 + else + printf "::debug::%s\n" "Found version ${tag_name} for DeepSource" ; + DS_TAG="${tag_name#v}" + fi ; + + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::debug::%s\n" "Creating ${BINDIR} directory for DeepSource" ; + mkdir -m 755 "${BINDIR}" || EXIT_CODE=66 ; + fi ; + + curl -fLso "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" "https://github.com/DeepSourceCorp/cli/releases/download/v${DS_TAG}/deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" || EXIT_CODE=125 ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Validated ds-cli.sh for DeepSource" ; - chmod -v 751 ./ds-cli.sh || EXIT_CODE=77 + printf "::debug::%s\n" "Downloaded deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; fi ; + curl -fLso "checksums.txt" "https://github.com/DeepSourceCorp/cli/releases/download/v${DS_TAG}/checksums.txt" || EXIT_CODE=125 ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Creating ${BINDIR} directory for DeepSource" ; - mkdir -v -m 755 "${BINDIR}" || EXIT_CODE=66 + printf "::debug::%s\n" "Downloaded checksums.txt for DeepSource" ; + fi ; + + if [[ ( -r checksums.txt ) ]] ; then + shasum -a 256 -c --ignore-missing checksums.txt || EXIT_CODE=126 + rm -f checksums.txt 2>/dev/null ; + fi ; + + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::debug::%s\n" "Validated deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; + tar --no-same-owner -xzf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" -C "${BINDIR}" || EXIT_CODE=77 ; + rm -f "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" 2>/dev/null || EXIT_CODE=77 ; + rm -fRd "${BINDIR}"/completions/ 2>/dev/null || : ; fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage - printf "::group::%s\n" "Running ./ds-cli.sh" ; - time { cat ./ds-cli.sh | sh ;} ; wait ; - printf "::endgroup::\n" ; + #printf "::group::%s\n" "Running ./ds-cli.sh" ; + #{ cat ./ds-cli.sh | sh - | grep -o "(:?version).{1}\s[0-9].[0-9].[0-9]" ;} ; wait ; + #printf "::endgroup::\n" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Checking for DeepSource CLI" ; if [[ ( -d "${BINDIR}" ) ]] ; then @@ -304,4 +333,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -b444b5d3e9093ee01e78a8fe991e60d0f6d48218fbb13cf8d3e6d06705a4289a7def2fcfe44758328c76572633cfd4ad - +779f220ba3f753ed3e929121e0742fe6362d779be1dfff7dda258e04b4fb95412e740b5f4ad5319b8ca61c79151cd735 - From 84f0d103b2817b167a3de9ab2f52065dfce9341c Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 20 Mar 2025 21:34:49 -0700 Subject: [PATCH 23/58] [DEBUG] Some fixes for untar permissions (- WIP PR #82 -) --- tests/fetch_cc-test-reporter | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fetch_cc-test-reporter b/tests/fetch_cc-test-reporter index 3c4b6db..ecd4a08 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch_cc-test-reporter @@ -272,14 +272,14 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Validated deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; - tar --no-same-owner -xzf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" -C "${BINDIR}" || EXIT_CODE=77 ; + tar --no-same-owner -xzf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" -C "${BINDIR}" 2>/dev/null || : ; rm -f "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" 2>/dev/null || EXIT_CODE=77 ; rm -fRd "${BINDIR}"/completions/ 2>/dev/null || : ; fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage - #printf "::group::%s\n" "Running ./ds-cli.sh" ; + printf "::debug::%s\n" "Extracted deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; #{ cat ./ds-cli.sh | sh - | grep -o "(:?version).{1}\s[0-9].[0-9].[0-9]" ;} ; wait ; #printf "::endgroup::\n" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then @@ -333,4 +333,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -779f220ba3f753ed3e929121e0742fe6362d779be1dfff7dda258e04b4fb95412e740b5f4ad5319b8ca61c79151cd735 - +f606b901a37e190d4d0a3e3f6945dda08598de00f182e212a3805c05e6110d5cde47fa492b549add4446124c14ed45f4 - From c24e2eda223c0ffafe65a188951f83e9f0bd6d5d Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 20 Mar 2025 23:46:44 -0700 Subject: [PATCH 24/58] [PATCH] Apply changes from Review (- WIP PR #82 -) Signed-off-by: Mr. Walls --- tests/check_cc_lines | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check_cc_lines b/tests/check_cc_lines index d8b1cbe..722272b 100755 --- a/tests/check_cc_lines +++ b/tests/check_cc_lines @@ -130,7 +130,7 @@ else EXIT_CODE=40 fi -_TEST_YEAR=$(date -j "+%C%y" 2>/dev/null ;) +_TEST_YEAR=$(date "+%Y" 2>/dev/null || date -j "+%C%y" 2>/dev/null ;) for _TEST_DOC in $(find ${_TEST_ROOT_DIR} \( -not -ipath "*.github/*" \) -a \( -iname '*.py' -o -iname '*.txt' -o -iname '*.md' \) -a -print0 2>&1 | xargs -0 -I{} ${_TEST_FILE_VALIDATOR} "{}" ; wait ;) ; do if [[ ($(grep -cF 'Disclaimer' "${_TEST_DOC}" 2>&1 || : ;) -ne 0) ]] ; then From 0708ad1806fd8b77adf4b11bc96a68388b6ca408 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 21 Mar 2025 16:46:23 -0700 Subject: [PATCH 25/58] [STYLE] apply changes found in review (- WIP pr #82 -) --- .github/workflows/Tests.yml | 4 +- Makefile | 4 +- ...h_cc-test-reporter => fetch-test-reporter} | 135 +++++++++++------- 3 files changed, 87 insertions(+), 56 deletions(-) rename tests/{fetch_cc-test-reporter => fetch-test-reporter} (73%) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 2dd85c4..0b5565a 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -199,7 +199,7 @@ jobs: - name: Install code-climate tools for ${{ matrix.python-version }} if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash - run: ./tests/fetch_cc-test-reporter || true ; + run: ./tests/fetch-test-reporter || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} if: ${{ runner.python-version == '3.9' }} run: make -f Makefile test ; @@ -304,7 +304,7 @@ jobs: pip install -r ./test-requirements.txt || true ; - name: Install code-climate tools for ${{ matrix.python-version }} on ${{ matrix.os }} if: ${{ !cancelled() && runner.os != 'Windows' }} - run: if [ $OS == macos-latest ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; + run: if [ $OS == macos-latest ] ; then ./tests/fetch-test-reporter || true ; fi ; shell: bash - name: Pre-Clean id: clean-prep diff --git a/Makefile b/Makefile index ba82364..8d7fede 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ else endif ifndef CC_TOOL - FETCH_CC_TOOL := tests/fetch_cc-test-reporter + FETCH_CC_TOOL := tests/fetch-test-reporter CC_TOOL := ./cc-test-reporter CC_TOOL_ARGS := after-build --exit-code 0 -t coverage.py DS_TOOL := ./bin/deepsource @@ -260,7 +260,7 @@ test-style: cleanup $(QUIET)tests/check_spelling 2>/dev/null || true $(QUIET)$(ECHO) "$@: Done." -cc-test-reporter: tests/fetch_cc-test-reporter +cc-test-reporter: tests/fetch-test-reporter $(QUIET)$(FETCH_CC_TOOL) || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; diff --git a/tests/fetch_cc-test-reporter b/tests/fetch-test-reporter similarity index 73% rename from tests/fetch_cc-test-reporter rename to tests/fetch-test-reporter index ecd4a08..8833fd7 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/fetch-test-reporter @@ -65,7 +65,7 @@ # to check though: -diff -q <(tail -n 269 "$0" | head -n 267 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; +diff -q <(tail -n 300 "$0" | head -n 298 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; ulimit -t 90 PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" @@ -76,17 +76,42 @@ DS_TAG="0.8.6" # circa 2025 umask 127 LOCK_FILE="${TMPDIR:-/tmp}/org.pak.tests.scripts.code-climate.lock" -EXIT_CODE=0 - -test -x "$(command -v grep)" || exit 126 ; -test -x "$(command -v sed)" || exit 126 ; -test -x "$(command -v curl)" || exit 126 ; -test -x "$(command -v mkdir)" || exit 126 ; -hash -p ./.github/tool_shlock_helper.sh shlock || exit 255 ; -test -x "$(command -v shlock)" || exit 126 ; -test -x "$(command -v gpgv)" || exit 126 ; -test -x "$(command -v shasum)" || exit 126 ; -test -x "$(command -v tar)" || exit 126 ; +EXIT_CODE=1 +# Function to check if a command exists. + +# USAGE: +# ~$ check_command CMD +# Arguments: +# CMD (Required) -- Name of the command to check +# Results: +# exits 64 -- missing required argument +# exits 126 -- check complete and has failed, can not find given command. +# returns successful -- check complete and command found to be executable. +function check_command() { + test -z "$1" && { printf "%s\n" "Error: command name is required to check for existence." >&2 ; exit 64 ; } ; + local cmd="$1" ; + # shellcheck disable=SC2086 + test -x "$(command -v ${cmd})" || { printf "%s\n" "Error: Required command '$cmd' is not found." >&2 ; exit 126 ; } ; +} # end check_command() +# propagate/export function to sub-shells +export -f check_command + +# Set up CEP-5 shlock helper +hash -p ./.github/tool_shlock_helper.sh shlock || { printf "%s\n" "Error: Failed to register shlock helper. CEP-5 locking will not work." >&2 ; exit 78 ; } ; +check_command cat ; +check_command chmod ; +check_command curl ; +check_command git ; +check_command grep ; +check_command gpg ; +check_command mkdir ; +check_command sed ; # but will try jq first +check_command shasum ; +check_command shlock ; +check_command tar ; +check_command uname ; + +SCRIPT_FILE="tests/fetch-test-reporter" # Detect the operating system case "$( command uname -s )" in @@ -140,24 +165,25 @@ function cleanup() { unset CI_ENV_PREFIX 2>/dev/null || : unset CI_OS 2>/dev/null || : unset ARCH 2>/dev/null || : + unset DS_TAG 2>/dev/null || : hash -d shlock 2>/dev/null > /dev/null || : ; } if [[ ( $(shlock -f "${LOCK_FILE}" -p $$ ) -eq 0 ) ]] ; then - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 - # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 - #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true || : ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 - trap 'cleanup ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 + EXIT_CODE=0 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 + # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 + #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 + trap 'cleanup ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 else - # shellcheck disable=SC2046 - printf "\t%s\n" "CodeClimate already in progress by "$(head "${LOCK_FILE}") >&2 ; - false ; - exit 126 ; + # shellcheck disable=SC2046 + printf "\t%s\n" "Fetch Tool already in progress by "$(head "${LOCK_FILE}") >&2 ; + exit 126 ; fi # this is how test files are found: @@ -187,7 +213,7 @@ fi ######################### # actual Work starts here ######################### -curl -fLso ./test-reporter-latest-${CI_ENV_PREFIX} https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX} ; +curl -fLso ./test-reporter-latest-${CI_ENV_PREFIX} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}" ; for i in 1 256 512 ; do curl -fLso test-reporter-latest-${CI_ENV_PREFIX}.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}.sha${i}" ; wait ; curl -fLso test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig" ; wait ; @@ -196,7 +222,7 @@ for i in 1 256 512 ; do if [[ ${i} -eq 1 ]]; then printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" fi - if [[ ( -r test-reporter-latest-${CI_OS:-linux}-${ARCH:-amd64}.sha${i}.sig ) ]] ; then + if [[ ( -r test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig ) ]] ; then # configure your CI evironment to trust the key at gpg --keyserver keys.openpgp.org --recv-keys 9BD9E2DD46DA965A537E5B0A5CBF320243B6FD85 # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 # OR... @@ -204,7 +230,7 @@ for i in 1 256 512 ; do gpg --verify test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=126 rm -vf test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig 2>/dev/null ; fi - shasum -a $i -c --ignore-missing test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || EXIT_CODE=126 + shasum -a $i -c --ignore-missing test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || EXIT_CODE=89 rm -vf test-reporter-latest-${CI_ENV_PREFIX}.sha${i} 2>/dev/null ; fi done @@ -215,7 +241,8 @@ if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then fi if [[ -z ${DEEPSOURCE_DSN} ]] && [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "%s\n" "No need for Unconfigured DeepSource." ; + printf "::notice %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=SKIP" \ + "No need for Unconfigured DeepSource." ; elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "%s\n" "Detected Config for DeepSource" ; @@ -224,8 +251,9 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # Check if the response is valid if [[ "$response" == *"Not Found"* ]]; then - printf "::debug::%s\n" "Repository not found or no releases available." - EXIT_CODE=125 + printf "::warning %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}" \ + "Repository not found or no releases available." >&2 ; + EXIT_CODE=69 fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then @@ -241,8 +269,9 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # Check if tag_name is empty if [[ ( -z "$tag_name" ) ]]; then - printf "::debug::%s\n" "No tag name found in the response." - EXIT_CODE=125 + printf "::warning %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}" \ + "No tag name found in the response." >&2 ; + EXIT_CODE=65 else printf "::debug::%s\n" "Found version ${tag_name} for DeepSource" ; DS_TAG="${tag_name#v}" @@ -250,7 +279,7 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Creating ${BINDIR} directory for DeepSource" ; - mkdir -m 755 "${BINDIR}" || EXIT_CODE=66 ; + mkdir -v -m 755 "${BINDIR}" || EXIT_CODE=66 ; fi ; curl -fLso "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" "https://github.com/DeepSourceCorp/cli/releases/download/v${DS_TAG}/deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" || EXIT_CODE=125 ; @@ -263,25 +292,29 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Downloaded checksums.txt for DeepSource" ; + if [[ ( -r checksums.txt ) ]] ; then + shasum -a 256 -c --ignore-missing checksums.txt || EXIT_CODE=89 + rm -vf checksums.txt 2>/dev/null || : ; + fi ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "%s\n" "Validated deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; + fi ; fi ; - - if [[ ( -r checksums.txt ) ]] ; then - shasum -a 256 -c --ignore-missing checksums.txt || EXIT_CODE=126 - rm -f checksums.txt 2>/dev/null ; - fi ; + + printf "::group::%s\n" "Extracting deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Validated deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; - tar --no-same-owner -xzf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" -C "${BINDIR}" 2>/dev/null || : ; - rm -f "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" 2>/dev/null || EXIT_CODE=77 ; - rm -fRd "${BINDIR}"/completions/ 2>/dev/null || : ; + tar --no-same-owner -xzf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" -C "${BINDIR}" || : ; + rm -vf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" 2>/dev/null || : ; + rm -vfRd "${BINDIR}"/completions/ 2>/dev/null || : ; + printf "::debug::%s\n" "Extracted deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; fi ; + printf "::endgroup::\n" ; + if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage printf "::debug::%s\n" "Extracted deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; - #{ cat ./ds-cli.sh | sh - | grep -o "(:?version).{1}\s[0-9].[0-9].[0-9]" ;} ; wait ; - #printf "::endgroup::\n" ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then printf "::debug::%s\n" "Checking for DeepSource CLI" ; if [[ ( -d "${BINDIR}" ) ]] ; then @@ -301,21 +334,19 @@ elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then else printf "::debug::..=> %s\n" "Missing ${BINDIR}/deepsource CLI tool" ; EXIT_CODE=66 ; - printf "::debug::..=> %s\n" "Checking if ${BINDIR} is empty:" ; - readonly BIN_FILES=$( ls -1 "${BINDIR}" 2>/dev/null ;) - printf "::debug::..=> %s\n" "${BIN_FILES}" ; - unset $BIN_FILES 2>/dev/null || : ; fi ; else printf "::debug::=> %s\n" "Missing ${BINDIR} directory." ; - EXIT_CODE=125 ; + EXIT_CODE=66 ; fi ; fi ; fi ; if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then + printf "::group::%s\n" "Checking DeepSource Config" ; # ref: https://docs.deepsource.com/docs/cli#commands - ./bin/deepsource config validate || EXIT_CODE=$? ; + ./bin/deepsource config validate || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=125 ; + printf "::endgroup::\n" ; fi ; wait ; fi ; @@ -333,4 +364,4 @@ cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait exit ${EXIT_CODE:-255} ; # This file's code hash: -f606b901a37e190d4d0a3e3f6945dda08598de00f182e212a3805c05e6110d5cde47fa492b549add4446124c14ed45f4 - +64775bab23e56bbd695b87b099c23978d789791c79e94ae7c5cd126a99ca7fde949260d9ad26297027db34af2022f436 - From 61117e11a10f056ac1a12f0cbb47258bf2f0eeb1 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 21 Mar 2025 17:05:18 -0700 Subject: [PATCH 26/58] [STYLE] Apply changes from review (- WIP PR #82 -) * fixes two minor typos in Makefile. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8d7fede..6ae158a 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,7 @@ MANIFEST.in: init build: init ./setup.py MANIFEST.in $(QUIET)$(PYTHON) -W ignore -m build --sdist --wheel --no-isolation ./ || $(QUIET)$(PYTHON) -W ignore -m build ./ ; - $(QUITE)$(WAIT) + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "build DONE." init: @@ -203,7 +203,7 @@ user-install: build uninstall: $(QUIET)$(PYTHON) -m pip uninstall --use-pep517 $(PIP_ENV_FLAGS) --no-input -y pythonrepo 2>$(ERROR_LOG_PATH) || : - $(QUITE)$(WAIT) + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "$@: Done." legacy-purge: clean uninstall From 4969152142c2befd6be04bdc3fd88ec4217602c2 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 24 Mar 2025 18:27:00 -0700 Subject: [PATCH 27/58] [CONFIG] refactoring fetch-test-reporter into submodule (- WIP PR #82 -) --- .gitmodules | 4 ++++ Makefile | 11 +++-------- 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4b7c06b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "fetch-test-reporter"] + path = includes/fetch-test-reporter + url = https://gist.github.com/c718380a1747d43dda1519167fc50ac0.git + ignore = dirty diff --git a/Makefile b/Makefile index 6ae158a..5113eaf 100644 --- a/Makefile +++ b/Makefile @@ -107,15 +107,8 @@ endif # Define environment-specific flags ifeq ($(shell uname -s), Darwin) PIP_ENV_FLAGS := --break-system-packages -else ifeq ($(shell uname -s), Linux) - PIP_ENV_FLAGS := else PIP_ENV_FLAGS := - FETCH_CC_TOOL := : - CC_TOOL := : - CC_TOOL_ARGS := - DS_TOOL := : - DS_TOOL_ARGS := endif ifeq "$(WAIT)" "" @@ -155,6 +148,8 @@ ifeq "$(RMDIR)" "" RMDIR=$(RM)Rd endif +include includes/fetch-test-reporter/tools-fetch-test-reporter.make + .PHONY: all clean test cleanup init help clean-docs must_be_root must_have_flake must_have_pytest uninstall cleanup-dev-backups @@ -260,7 +255,7 @@ test-style: cleanup $(QUIET)tests/check_spelling 2>/dev/null || true $(QUIET)$(ECHO) "$@: Done." -cc-test-reporter: tests/fetch-test-reporter +cc-test-reporter: $(FETCH_CC_TOOL) $(QUIET)$(FETCH_CC_TOOL) || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; From c79061aee4bceebd47022f1181ccfa9f06842e74 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 24 Mar 2025 18:30:44 -0700 Subject: [PATCH 28/58] [CONFIG] Part two of refactor (- WIP #79 -) --- includes/fetch-test-reporter | 1 + tests/fetch-test-reporter | 368 +---------------------------------- 2 files changed, 2 insertions(+), 367 deletions(-) create mode 160000 includes/fetch-test-reporter mode change 100755 => 120000 tests/fetch-test-reporter diff --git a/includes/fetch-test-reporter b/includes/fetch-test-reporter new file mode 160000 index 0000000..4fc9a02 --- /dev/null +++ b/includes/fetch-test-reporter @@ -0,0 +1 @@ +Subproject commit 4fc9a026110a0ac52b5bbc2a499d0da01bc7ce7b diff --git a/tests/fetch-test-reporter b/tests/fetch-test-reporter deleted file mode 100755 index 8833fd7..0000000 --- a/tests/fetch-test-reporter +++ /dev/null @@ -1,367 +0,0 @@ -#! /bin/bash -# Disclaimer of Warranties. -# A. YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT, TO THE EXTENT PERMITTED BY -# APPLICABLE LAW, USE OF THIS SHELL SCRIPT AND ANY SERVICES PERFORMED -# BY OR ACCESSED THROUGH THIS SHELL SCRIPT IS AT YOUR SOLE RISK AND -# THAT THE ENTIRE RISK AS TO SATISFACTORY QUALITY, PERFORMANCE, ACCURACY AND -# EFFORT IS WITH YOU. -# -# B. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THIS SHELL SCRIPT -# AND SERVICES ARE PROVIDED "AS IS" AND “AS AVAILABLE”, WITH ALL FAULTS AND -# WITHOUT WARRANTY OF ANY KIND, AND THE AUTHOR OF THIS SHELL SCRIPT'S LICENSORS -# (COLLECTIVELY REFERRED TO AS "THE AUTHOR" FOR THE PURPOSES OF THIS DISCLAIMER) -# HEREBY DISCLAIM ALL WARRANTIES AND CONDITIONS WITH RESPECT TO THIS SHELL SCRIPT -# SOFTWARE AND SERVICES, EITHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT -# NOT LIMITED TO, THE IMPLIED WARRANTIES AND/OR CONDITIONS OF -# MERCHANTABILITY, SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE, -# ACCURACY, QUIET ENJOYMENT, AND NON-INFRINGEMENT OF THIRD PARTY RIGHTS. -# -# C. THE AUTHOR DOES NOT WARRANT AGAINST INTERFERENCE WITH YOUR ENJOYMENT OF THE -# THE AUTHOR's SOFTWARE AND SERVICES, THAT THE FUNCTIONS CONTAINED IN, OR -# SERVICES PERFORMED OR PROVIDED BY, THIS SHELL SCRIPT WILL MEET YOUR -# REQUIREMENTS, THAT THE OPERATION OF THIS SHELL SCRIPT OR SERVICES WILL -# BE UNINTERRUPTED OR ERROR-FREE, THAT ANY SERVICES WILL CONTINUE TO BE MADE -# AVAILABLE, THAT THIS SHELL SCRIPT OR SERVICES WILL BE COMPATIBLE OR -# WORK WITH ANY THIRD PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES, -# OR THAT DEFECTS IN THIS SHELL SCRIPT OR SERVICES WILL BE CORRECTED. -# INSTALLATION OF THIS THE AUTHOR SOFTWARE MAY AFFECT THE USABILITY OF THIRD -# PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES. -# -# D. YOU FURTHER ACKNOWLEDGE THAT THIS SHELL SCRIPT AND SERVICES ARE NOT -# INTENDED OR SUITABLE FOR USE IN SITUATIONS OR ENVIRONMENTS WHERE THE FAILURE -# OR TIME DELAYS OF, OR ERRORS OR INACCURACIES IN, THE CONTENT, DATA OR -# INFORMATION PROVIDED BY THIS SHELL SCRIPT OR SERVICES COULD LEAD TO -# DEATH, PERSONAL INJURY, OR SEVERE PHYSICAL OR ENVIRONMENTAL DAMAGE, -# INCLUDING WITHOUT LIMITATION THE OPERATION OF NUCLEAR FACILITIES, AIRCRAFT -# NAVIGATION OR COMMUNICATION SYSTEMS, AIR TRAFFIC CONTROL, LIFE SUPPORT OR -# WEAPONS SYSTEMS. -# -# E. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY THE AUTHOR -# SHALL CREATE A WARRANTY. SHOULD THIS SHELL SCRIPT OR SERVICES PROVE DEFECTIVE, -# YOU ASSUME THE ENTIRE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. -# -# Limitation of Liability. -# F. TO THE EXTENT NOT PROHIBITED BY APPLICABLE LAW, IN NO EVENT SHALL THE AUTHOR -# BE LIABLE FOR PERSONAL INJURY, OR ANY INCIDENTAL, SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES WHATSOEVER, INCLUDING, WITHOUT LIMITATION, DAMAGES -# FOR LOSS OF PROFITS, CORRUPTION OR LOSS OF DATA, FAILURE TO TRANSMIT OR -# RECEIVE ANY DATA OR INFORMATION, BUSINESS INTERRUPTION OR ANY OTHER -# COMMERCIAL DAMAGES OR LOSSES, ARISING OUT OF OR RELATED TO YOUR USE OR -# INABILITY TO USE THIS SHELL SCRIPT OR SERVICES OR ANY THIRD PARTY -# SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH THIS SHELL SCRIPT OR -# SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY OF LIABILITY (CONTRACT, -# TORT OR OTHERWISE) AND EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION -# OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF INCIDENTAL OR -# CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU. In no event -# shall THE AUTHOR's total liability to you for all damages (other than as may -# be required by applicable law in cases involving personal injury) exceed -# the amount of five dollars ($5.00). The foregoing limitations will apply -# even if the above stated remedy fails of its essential purpose. -################################################################################ - -# CAVEAT: This Script is for convience and should not be used as a source of truth -# when checking crypto hashes. - - -# to check though: -diff -q <(tail -n 300 "$0" | head -n 298 | shasum -a 384 -t -) <(tail -n 1 "$0") || exit 70 ; - -ulimit -t 90 -PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" -LANG=${LANG:-"en_US"} -LC_ALL="${LANG:0:5}.utf-8" -BINDIR="${BINDIR:-./bin}" -DS_TAG="0.8.6" # circa 2025 -umask 127 - -LOCK_FILE="${TMPDIR:-/tmp}/org.pak.tests.scripts.code-climate.lock" -EXIT_CODE=1 -# Function to check if a command exists. - -# USAGE: -# ~$ check_command CMD -# Arguments: -# CMD (Required) -- Name of the command to check -# Results: -# exits 64 -- missing required argument -# exits 126 -- check complete and has failed, can not find given command. -# returns successful -- check complete and command found to be executable. -function check_command() { - test -z "$1" && { printf "%s\n" "Error: command name is required to check for existence." >&2 ; exit 64 ; } ; - local cmd="$1" ; - # shellcheck disable=SC2086 - test -x "$(command -v ${cmd})" || { printf "%s\n" "Error: Required command '$cmd' is not found." >&2 ; exit 126 ; } ; -} # end check_command() -# propagate/export function to sub-shells -export -f check_command - -# Set up CEP-5 shlock helper -hash -p ./.github/tool_shlock_helper.sh shlock || { printf "%s\n" "Error: Failed to register shlock helper. CEP-5 locking will not work." >&2 ; exit 78 ; } ; -check_command cat ; -check_command chmod ; -check_command curl ; -check_command git ; -check_command grep ; -check_command gpg ; -check_command mkdir ; -check_command sed ; # but will try jq first -check_command shasum ; -check_command shlock ; -check_command tar ; -check_command uname ; - -SCRIPT_FILE="tests/fetch-test-reporter" - -# Detect the operating system -case "$( command uname -s )" in - *arwin) - CI_OS="darwin" - ;; - Linux) - CI_OS="linux" - ;; - *) - # sorry no windows support here - printf 'Unsupported OS\n' >&2 ; - exit 75 ; # exit code for temporary failure, as per CEP-8 - ;; -esac - -# Detect the architecture -ARCH="$(command uname -m)" -case "$ARCH" in - x86_64) - ARCH="amd64" - ;; - arm64|aarch64|armv8l) - ARCH="arm64" - ;; - armv7l|armv6l) - ARCH="arm" - ;; - *) - # sorry no windows support here - printf 'Unsupported architecture\n' >&2 ; - exit 75 ; - ;; -esac - -# Set the CI prefix -case "${CI_OS:-Linux}" in - *arwin) - CI_ENV_PREFIX="${CI_OS}-amd64" - ;; - *) - CI_ENV_PREFIX="${CI_OS}-${ARCH}" - ;; -esac - -function cleanup() { - rm -f "${LOCK_FILE}" 2>/dev/null || : ; wait ; - rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.sha*.sig 2>/dev/null || : ; wait ; - rm -f ./test-reporter-latest-${CI_ENV_PREFIX}.sha* 2>/dev/null > /dev/null || : ; wait ; - rm -f ./ds-cli.sh.sha* 2>/dev/null > /dev/null || : ; wait ; - unset CI_ENV_PREFIX 2>/dev/null || : - unset CI_OS 2>/dev/null || : - unset ARCH 2>/dev/null || : - unset DS_TAG 2>/dev/null || : - hash -d shlock 2>/dev/null > /dev/null || : ; -} - -if [[ ( $(shlock -f "${LOCK_FILE}" -p $$ ) -eq 0 ) ]] ; then - EXIT_CODE=0 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 - # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 - #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 - trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || : ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 - trap 'cleanup ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 -else - # shellcheck disable=SC2046 - printf "\t%s\n" "Fetch Tool already in progress by "$(head "${LOCK_FILE}") >&2 ; - exit 126 ; -fi - -# this is how test files are found: - -# THIS IS THE ACTUAL TEST DIR USED (update _TEST_ROOT_DIR as needed) -_TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) ; -if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); then - if [ -z "${_TEST_ROOT_DIR}" ]; then - _TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) - fi - printf "::debug::%s\n" "Found ${_TEST_ROOT_DIR} ..." ; -else - printf "::error %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}" \ - "FAIL: missing valid repository or source structure" >&2 ; - EXIT_CODE=40 -fi - -# This File MUST BE GIT-IGNORED -# to be SAFELY USED to store Tokens and env vars (update logic as needed) -if [[ ( -r ./codecov_env ) ]] ; then - printf "::debug::%s\n" "Found codecov_env ..." ; - # shellcheck disable=SC2086,SC1091 - source ./codecov_env 2>/dev/null || : ; -fi - - -######################### -# actual Work starts here -######################### -curl -fLso ./test-reporter-latest-${CI_ENV_PREFIX} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}" ; -for i in 1 256 512 ; do - curl -fLso test-reporter-latest-${CI_ENV_PREFIX}.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}.sha${i}" ; wait ; - curl -fLso test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig" ; wait ; - # test sha1/sha512 signatures if found and sha256 even if not found - if [[ ( -r test-reporter-latest-${CI_ENV_PREFIX}.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then - if [[ ${i} -eq 1 ]]; then - printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" - fi - if [[ ( -r test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig ) ]] ; then - # configure your CI evironment to trust the key at gpg --keyserver keys.openpgp.org --recv-keys 9BD9E2DD46DA965A537E5B0A5CBF320243B6FD85 - # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 - # OR... - # Set CI=true to continue on missing keys - gpg --verify test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_ENV_PREFIX}.sha${i}.sig 2>/dev/null ; - fi - shasum -a $i -c --ignore-missing test-reporter-latest-${CI_ENV_PREFIX}.sha${i} || EXIT_CODE=89 - rm -vf test-reporter-latest-${CI_ENV_PREFIX}.sha${i} 2>/dev/null ; - fi -done - -if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - mv -fv ./test-reporter-latest-${CI_ENV_PREFIX} ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 - chmod -v 751 ./cc-test-reporter || EXIT_CODE=77 -fi - -if [[ -z ${DEEPSOURCE_DSN} ]] && [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::notice %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=SKIP" \ - "No need for Unconfigured DeepSource." ; -elif [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "%s\n" "Detected Config for DeepSource" ; - - # Make an API call to GitHub to get the latest release with the Accept header - response=$(curl -s -H "Accept: application/json" "https://api.github.com/repos/DeepSourceCorp/cli/releases/latest") - - # Check if the response is valid - if [[ "$response" == *"Not Found"* ]]; then - printf "::warning %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}" \ - "Repository not found or no releases available." >&2 ; - EXIT_CODE=69 - fi ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Checking latest version for DeepSource" ; - # Try to extract the tag name using jq - if [[ ( -x "$(command -v jq)" ) ]]; then - tag_name=$(printf "%s\n" "$response" | jq -r '.tag_name') - else - # Fallback to grep and sed if jq is not available - tag_name=$(printf "%s\n" "$response" | grep -o '"tag_name": "[^"]*' | sed 's/"tag_name": "//') - fi ; - fi ; - - # Check if tag_name is empty - if [[ ( -z "$tag_name" ) ]]; then - printf "::warning %s::%s\n" "file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}" \ - "No tag name found in the response." >&2 ; - EXIT_CODE=65 - else - printf "::debug::%s\n" "Found version ${tag_name} for DeepSource" ; - DS_TAG="${tag_name#v}" - fi ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Creating ${BINDIR} directory for DeepSource" ; - mkdir -v -m 755 "${BINDIR}" || EXIT_CODE=66 ; - fi ; - - curl -fLso "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" "https://github.com/DeepSourceCorp/cli/releases/download/v${DS_TAG}/deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" || EXIT_CODE=125 ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Downloaded deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; - fi ; - - curl -fLso "checksums.txt" "https://github.com/DeepSourceCorp/cli/releases/download/v${DS_TAG}/checksums.txt" || EXIT_CODE=125 ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Downloaded checksums.txt for DeepSource" ; - if [[ ( -r checksums.txt ) ]] ; then - shasum -a 256 -c --ignore-missing checksums.txt || EXIT_CODE=89 - rm -vf checksums.txt 2>/dev/null || : ; - fi ; - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "%s\n" "Validated deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz for DeepSource" ; - fi ; - fi ; - - printf "::group::%s\n" "Extracting deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - tar --no-same-owner -xzf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" -C "${BINDIR}" || : ; - rm -vf "deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" 2>/dev/null || : ; - rm -vfRd "${BINDIR}"/completions/ 2>/dev/null || : ; - printf "::debug::%s\n" "Extracted deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; - fi ; - - printf "::endgroup::\n" ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - # ref: https://docs.deepsource.com/docs/analyzers-test-coverage#setup-test-coverage - printf "::debug::%s\n" "Extracted deepsource_${DS_TAG}_${CI_OS}_${ARCH}.tar.gz" ; - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::debug::%s\n" "Checking for DeepSource CLI" ; - if [[ ( -d "${BINDIR}" ) ]] ; then - printf "::debug::=> %s\n" "Found ${BINDIR} directory." ; - if [[ ( -x "${BINDIR}"/deepsource ) ]] ; then - printf "::debug::..=> %s\n" "Found ${BINDIR}/deepsource CLI tool" ; - elif [[ ( -r "${BINDIR}"/deepsource ) || ( -e "${BINDIR}"/deepsource ) ]] ; then - printf "::debug::..=> %s\n" "Found ${BINDIR}/deepsource file." ; - printf "::debug::....=> %s\n" "Trying to apply permission corrections" ; - chmod -v 751 "${BINDIR}"/deepsource || EXIT_CODE=77 ; - if [[ ( ${EXIT_CODE} -eq 0 ) ]] && [[ ( -x "${BINDIR}"/deepsource ) ]] ; then - printf "::debug::......=> %s\n" "Fixed ${BINDIR}/deepsource CLI tool" ; - else - printf "::debug::....=> %s\n" "Applying corrections Unsuccessful" ; - EXIT_CODE=77 ; - fi ; - else - printf "::debug::..=> %s\n" "Missing ${BINDIR}/deepsource CLI tool" ; - EXIT_CODE=66 ; - fi ; - else - printf "::debug::=> %s\n" "Missing ${BINDIR} directory." ; - EXIT_CODE=66 ; - fi ; - fi ; - fi ; - - if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - printf "::group::%s\n" "Checking DeepSource Config" ; - # ref: https://docs.deepsource.com/docs/cli#commands - ./bin/deepsource config validate || ${CI} 2>/dev/null || ${CIRCLECI} 2>/dev/null || EXIT_CODE=125 ; - printf "::endgroup::\n" ; - fi ; - wait ; -fi ; - -if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - ./cc-test-reporter before-build || EXIT_CODE=125 ; -fi - - -unset _TEST_ROOT_DIR 2>/dev/null || : ; - -cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || : ; wait ; - -# goodbye -exit ${EXIT_CODE:-255} ; - -# This file's code hash: -64775bab23e56bbd695b87b099c23978d789791c79e94ae7c5cd126a99ca7fde949260d9ad26297027db34af2022f436 - diff --git a/tests/fetch-test-reporter b/tests/fetch-test-reporter new file mode 120000 index 0000000..b718172 --- /dev/null +++ b/tests/fetch-test-reporter @@ -0,0 +1 @@ +../includes/fetch-test-reporter/fetch-test-reporter \ No newline at end of file From 5d9deddfa0c93169dbead34e882694c0fdfda72c Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 24 Mar 2025 23:00:37 -0700 Subject: [PATCH 29/58] [PATCH] Update Submodule (- WIP #79 -) --- includes/fetch-test-reporter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/fetch-test-reporter b/includes/fetch-test-reporter index 4fc9a02..9dd2429 160000 --- a/includes/fetch-test-reporter +++ b/includes/fetch-test-reporter @@ -1 +1 @@ -Subproject commit 4fc9a026110a0ac52b5bbc2a499d0da01bc7ce7b +Subproject commit 9dd242932cc02372b703645e99728b5a466b959d From c8f329f266920fab03f85f522bb28a877affb1e0 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 24 Mar 2025 23:13:28 -0700 Subject: [PATCH 30/58] [CONFIG] Fix for GHA checkout for submodules (- WIP #79 -) --- .github/workflows/Tests.yml | 21 +++++++++++++++++++++ .github/workflows/codeql-analysis.yml | 5 ++++- .github/workflows/scorecards.yml | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 0b5565a..4d3dfad 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -28,6 +28,9 @@ jobs: LANG: "en_US.UTF-8" steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.12" @@ -75,6 +78,9 @@ jobs: LANG: ${{ matrix.lang-var }} steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 with: @@ -134,6 +140,9 @@ jobs: LANG: "en_US.utf-8" steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -181,6 +190,9 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -249,6 +261,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 with: @@ -293,6 +308,9 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -362,6 +380,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5439c0f..d243379 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,7 +41,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.2 + with: + submodules: true + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 99a4e23..03061c2 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -34,6 +34,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: + submodules: true persist-credentials: false - name: "Run analysis" From f84cbb732f8a9dea2b41bdf65557d3f774a6fd89 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 24 Mar 2025 23:26:53 -0700 Subject: [PATCH 31/58] [PATCH] Fix for cross-plaform import path support (- WIP #79 -) --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5113eaf..530013e 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,11 @@ endif # Define environment-specific flags ifeq ($(shell uname -s), Darwin) PIP_ENV_FLAGS := --break-system-packages + INCLUDE_PATH = includes/fetch-test-reporter/tools-fetch-test-reporter.make +else ifeq ($(shell uname -s), Linux) + INCLUDE_PATH = includes/fetch-test-reporter/tools-fetch-test-reporter.make else + INCLUDE_PATH = includes\fetch-test-reporter\tools-fetch-test-reporter.make PIP_ENV_FLAGS := endif @@ -148,7 +152,8 @@ ifeq "$(RMDIR)" "" RMDIR=$(RM)Rd endif -include includes/fetch-test-reporter/tools-fetch-test-reporter.make +# Include the makefile +include $(INCLUDE_PATH) .PHONY: all clean test cleanup init help clean-docs must_be_root must_have_flake must_have_pytest uninstall cleanup-dev-backups From 2e29e57f8ba850152eb85da18f38ddbdfa2563ae Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 24 Mar 2025 23:35:08 -0700 Subject: [PATCH 32/58] [PATCH] Fix to work around windows braindeath state (- WIP #79 -) --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 530013e..45805cf 100644 --- a/Makefile +++ b/Makefile @@ -107,11 +107,11 @@ endif # Define environment-specific flags ifeq ($(shell uname -s), Darwin) PIP_ENV_FLAGS := --break-system-packages - INCLUDE_PATH = includes/fetch-test-reporter/tools-fetch-test-reporter.make + FETCH_CC_INCLUDE_PATH := includes/fetch-test-reporter/tools-fetch-test-reporter.make else ifeq ($(shell uname -s), Linux) - INCLUDE_PATH = includes/fetch-test-reporter/tools-fetch-test-reporter.make + FETCH_CC_INCLUDE_PATH := includes/fetch-test-reporter/tools-fetch-test-reporter.make else - INCLUDE_PATH = includes\fetch-test-reporter\tools-fetch-test-reporter.make + FETCH_CC_INCLUDE_PATH := includes\fetch-test-reporter\tools-fetch-test-reporter.make PIP_ENV_FLAGS := endif @@ -153,7 +153,7 @@ ifeq "$(RMDIR)" "" endif # Include the makefile -include $(INCLUDE_PATH) +include $(FETCH_CC_INCLUDE_PATH) .PHONY: all clean test cleanup init help clean-docs must_be_root must_have_flake must_have_pytest uninstall cleanup-dev-backups @@ -260,7 +260,7 @@ test-style: cleanup $(QUIET)tests/check_spelling 2>/dev/null || true $(QUIET)$(ECHO) "$@: Done." -cc-test-reporter: $(FETCH_CC_TOOL) +cc-test-reporter: $(FETCH_CC_INCLUDE_PATH) $(QUIET)$(FETCH_CC_TOOL) || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; From e078879b94a1c8e3d119d0e89cb6358bb74955f7 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Tue, 25 Mar 2025 16:45:43 -0700 Subject: [PATCH 33/58] [PATCH] apply changes from review (- WIP PR #82 -) --- .github/workflows/Tests.yml | 11 +++--- Makefile | 69 ++++++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 4d3dfad..01ab20f 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -4,6 +4,8 @@ on: branches: - master - stable + - patch* + - feature* tags: - v* pull_request: @@ -11,7 +13,6 @@ on: - opened - edited - reopened - - synchronize - ready_for_review # Declare default permissions as read only. @@ -134,7 +135,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - python-version: ["3.9.20", "3.10", "3.11", "3.12"] + python-version: ["3.9.21", "3.10", "3.11", "3.12"] env: PYTHON_VERSION: ${{ matrix.python-version }} LANG: "en_US.utf-8" @@ -213,10 +214,10 @@ jobs: shell: bash run: ./tests/fetch-test-reporter || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} - if: ${{ runner.python-version == '3.9' }} + if: ${{ matrix.python-version == '3.9' }} run: make -f Makefile test ; - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} - if: ${{ runner.python-version != '3.9' }} + if: ${{ matrix.python-version != '3.9' }} run: make -f Makefile test-pytest ; - name: Upload Python ${{ matrix.python-version }} coverage to Codecov uses: codecov/codecov-action@v5 @@ -238,7 +239,7 @@ jobs: if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash run: | - if [ "$OS" == "ubuntu-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; + ./cc-test-reporter after-build --exit-code 0 || true ; - name: deepsource for ${{ matrix.python-version }} if: ${{ runner.os }} == "Linux" shell: bash diff --git a/Makefile b/Makefile index 45805cf..31be5dc 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,17 @@ ifeq "$(ERROR_LOG_PATH)" "" endif ifeq "$(COMMAND)" "" - COMMAND_CMD!=`command -v xcrun || command which which || command -v which || command -v command` + COMMAND_CMD!=`command -v xcrun || command -v command || command which which || command -v which` ifeq "$(COMMAND_CMD)" "*xcrun" - COMMAND_ARGS=--find + COMMAND_ARGS= --find endif ifeq "$(COMMAND_CMD)" "*command" - COMMAND_ARGS=-pv + COMMAND_ARGS= -pv endif - COMMAND=$(COMMAND_CMD) $(COMMAND_ARGS) + ifeq "$(COMMAND_CMD)" "" + COMMAND_CMD="command" + endif + COMMAND := $(COMMAND_CMD)$(COMMAND_ARGS) endif ifeq "$(MAKE)" "" @@ -62,43 +65,41 @@ else endif ifeq "$(LINK)" "" - LINK=ln -sf + LINK=$(COMMAND) ln -sf endif +# Python command configuration ifeq "$(PYTHON)" "" + # Try to find python3, fallback to python PY_CMD=$(COMMAND) python3 ifneq "$(PY_CMD)" "" + # Only use -B arg with python3 PY_ARGS=-B else PY_CMD=$(COMMAND) python endif - PYTHON=$(PY_CMD) $(PY_ARGS) - ifeq "$(COVERAGE)" "" + # Set PYTHON only if not already set + PYTHON := $(PY_CMD) $(PY_ARGS) +endif + +# Coverage configuration +ifeq "$(COVERAGE)" "" + # If PYTHON is set (either by us or externally), try module approach first + ifneq "$(PYTHON)" "" COVERAGE=$(PYTHON) -m coverage - #COV_CORE_SOURCE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/ - COV_CORE_CONFIG = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.coveragerc - COV_CORE_DATAFILE = .coverage endif + # If COVERAGE is still not set, fall back to direct command ifeq "$(COVERAGE)" "" COVERAGE!=$(COMMAND) coverage endif -else - ifeq "$(COVERAGE)" "" - COVERAGE!=$(COMMAND) coverage + # Only set COV_CORE_* variables when COVERAGE is configured + ifneq "$(COVERAGE)" "" #COV_CORE_SOURCE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/ COV_CORE_CONFIG = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.coveragerc COV_CORE_DATAFILE = .coverage endif endif -ifndef CC_TOOL - FETCH_CC_TOOL := tests/fetch-test-reporter - CC_TOOL := ./cc-test-reporter - CC_TOOL_ARGS := after-build --exit-code 0 -t coverage.py - DS_TOOL := ./bin/deepsource - DS_TOOL_ARGS := report --analyzer test-coverage --key python --value-file ./coverage.xml -endif - ifndef PIP_COMMON_FLAGS # Define common pip install flags PIP_COMMON_FLAGS := --use-pep517 --exists-action s --upgrade --upgrade-strategy eager @@ -157,6 +158,26 @@ include $(FETCH_CC_INCLUDE_PATH) .PHONY: all clean test cleanup init help clean-docs must_be_root must_have_flake must_have_pytest uninstall cleanup-dev-backups +.env: + $(QUIET)$(ECHO) "Missing environment override." + $(QUIET)$(ECHO) " SHELL: '$(SHELL)'" + $(QUIET)$(ECHO) " LOG: '$(LOG)'" + $(QUIET)$(ECHO) " ERROR_LOG_PATH: '$(ERROR_LOG_PATH)'" + $(QUIET)$(ECHO) " QUIET: '$(QUIET)'" + $(QUIET)$(ECHO) " COMMAND: '$(COMMAND)'" + $(QUIET)$(ECHO) " MAKE: '$(MAKE)'" + $(QUIET)$(ECHO) " ECHO: '$(ECHO)'" + $(QUIET)$(ECHO) " PYTHON: '$(PYTHON)'" + $(QUIET)$(ECHO) " COVERAGE: '$(COVERAGE)'" + $(QUIET)$(ECHO) " FETCH_CC_INCLUDE_PATH: '$(FETCH_CC_INCLUDE_PATH)'" + $(QUIET)$(ECHO) " CC_TOOL: '$(CC_TOOL)'" + $(QUIET)$(ECHO) " DS_TOOL: '$(DS_TOOL)'" + $(QUIET)$(ECHO) " CC_TOOL_ARGS: '$(CC_TOOL_ARGS)'" + $(QUIET)$(ECHO) " DS_TOOL_ARGS: '$(DS_TOOL_ARGS)'" + $(QUIET)$(ECHO) " PIP_ENV_FLAGS: '$(PIP_ENV_FLAGS)'" + $(QUIET)$(ECHO) " PIP_COMMON_FLAGS: '$(PIP_COMMON_FLAGS)'" + $(QUIET)$(ECHO) " DO_FAIL: '$(DO_FAIL)'" + $(QUIET)$(ECHO) "" # intentionally blank MANIFEST.in: init $(QUIET)$(ECHO) "include requirements.txt" >"$@" ; @@ -222,15 +243,15 @@ purge: legacy-purge $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." -test-reports: +test-reports: .env $(QUIET)mkdir $(INST_OPTS) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(BSMARK) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(ECHO) "$@: Done." test-reqs: cc-test-reporter test-reports init - $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r tests/requirements.txt 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r tests-requirements.txt 2>$(ERROR_LOG_PATH) || true -just-test: cleanup +just-test: cleanup .env $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s ./tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s ./tests -t ./ || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; From 608234f618d0f419fd4944c0c1e646c1e300bb15 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 26 Mar 2025 14:40:11 -0700 Subject: [PATCH 34/58] [PATCH] Update submodule (- WIP #79 -) * update submodule for fetch-cc-reporter * related work --- Makefile | 1 + includes/fetch-test-reporter | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 31be5dc..9f71549 100644 --- a/Makefile +++ b/Makefile @@ -170,6 +170,7 @@ include $(FETCH_CC_INCLUDE_PATH) $(QUIET)$(ECHO) " PYTHON: '$(PYTHON)'" $(QUIET)$(ECHO) " COVERAGE: '$(COVERAGE)'" $(QUIET)$(ECHO) " FETCH_CC_INCLUDE_PATH: '$(FETCH_CC_INCLUDE_PATH)'" + $(QUIET)$(ECHO) " FETCH_CC_TOOL: '$(FETCH_CC_TOOL)'" $(QUIET)$(ECHO) " CC_TOOL: '$(CC_TOOL)'" $(QUIET)$(ECHO) " DS_TOOL: '$(DS_TOOL)'" $(QUIET)$(ECHO) " CC_TOOL_ARGS: '$(CC_TOOL_ARGS)'" diff --git a/includes/fetch-test-reporter b/includes/fetch-test-reporter index 9dd2429..87fdabd 160000 --- a/includes/fetch-test-reporter +++ b/includes/fetch-test-reporter @@ -1 +1 @@ -Subproject commit 9dd242932cc02372b703645e99728b5a466b959d +Subproject commit 87fdabdae05040063a2cfef3f9275c5832f122e0 From ee15b1e120f9a895cd990bda96d980729900c11c Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 26 Mar 2025 18:34:39 -0700 Subject: [PATCH 35/58] [CONFIG] refactor circle-CI to handle submodules (- WIP #79 -) --- .circleci/config.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1bf887..91344bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,32 @@ commands: command: | make clean when: always + reposync: + steps: + - run: + shell: /bin/bash + name: "Fetch" + command: | + git fetch ; + when: always + - run: + shell: /bin/bash + name: "Pull" + command: | + git pull --all ; + when: on_success + - run: + shell: /bin/bash + name: "Sync Submodules" + command: | + git submodule sync ; + when: on_success + - run: + shell: /bin/bash + name: "Update Submodules" + command: | + git submodule update --init ; + when: on_success parameters: python-version: @@ -29,10 +55,7 @@ jobs: working_directory: ~/python-repo steps: - checkout - - run: - name: "fetch and pull" - command: | - git fetch && git pull --all || : ; + - reposync - run: shell: /bin/bash name: "install depends attempt" From 13d945198b3de3a955c8193cfcabfbbbe745eaa6 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 26 Mar 2025 18:45:01 -0700 Subject: [PATCH 36/58] [CONFIG] Regression fixes for CircleCI submodule support changes (- WIP #79 -) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91344bd..0f97118 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ commands: shell: /bin/bash name: "Fetch" command: | - git fetch ; + git fetch --unshallow --recurse-submodules=yes --all ; when: always - run: shell: /bin/bash From 2680121a885e7810cd13a595c46a7a098a31931d Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 26 Mar 2025 18:48:05 -0700 Subject: [PATCH 37/58] [CONFIG] More tweaks for CircleCI submodule support (- WIP #79 -) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f97118..286e2b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ commands: shell: /bin/bash name: "Fetch" command: | - git fetch --unshallow --recurse-submodules=yes --all ; + git fetch --verbose --all ; when: always - run: shell: /bin/bash From dc9c842de96f1644b16dea5035afd20d8504b786 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 26 Mar 2025 18:52:58 -0700 Subject: [PATCH 38/58] [CONFIG] Refactor to skip extra pull by CircleCI (- WIP #79 -) --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 286e2b3..3e0a6e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,14 +15,8 @@ commands: shell: /bin/bash name: "Fetch" command: | - git fetch --verbose --all ; + git fetch --all ; when: always - - run: - shell: /bin/bash - name: "Pull" - command: | - git pull --all ; - when: on_success - run: shell: /bin/bash name: "Sync Submodules" From d4cef6266e17a9801a3aafa4ce03441e75c1ec85 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 27 Mar 2025 00:50:37 -0700 Subject: [PATCH 39/58] [PATCH] implemented PoC support for coveralls coverage reporter via submodule tool (- WIP #79 -) --- .circleci/config.yml | 2 +- Makefile | 13 ++++++++----- includes/fetch-test-reporter | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e0a6e4..144e304 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,7 +120,7 @@ jobs: when: on_success - run: shell: /bin/bash - name: "clean up for test" + name: "install test-reqs attempt" command: | python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; when: on_success diff --git a/Makefile b/Makefile index 9f71549..5ef1974 100644 --- a/Makefile +++ b/Makefile @@ -252,12 +252,12 @@ test-reports: .env test-reqs: cc-test-reporter test-reports init $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r tests-requirements.txt 2>$(ERROR_LOG_PATH) || true -just-test: cleanup .env +just-test: cleanup cc-test-reporter .env $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s ./tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s ./tests -t ./ || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; -test: just-test cc-test-reporter +test: just-test $(QUIET)$(DO_FAIL) ; $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; @@ -270,8 +270,9 @@ test-tox: cleanup test-pytest: cleanup MANIFEST.in cc-test-reporter must_have_pytest test-reports $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=. || DO_FAIL="exit 2" ; - $(QUIET)$(DS_TOOL) $(DS_TOOL_ARGS) || : ; $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(CA_TOOL) $(CA_TOOL_ARGS) || : ; + $(QUIET)$(DS_TOOL) $(DS_TOOL_ARGS) || : ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." @@ -296,6 +297,9 @@ must_have_pytest: init $(QUIET)runner=`$(PYTHON) -m pip freeze --all | grep --count -oF pytest` ; \ if test $$runner -le 0 ; then $(ECHO) "No python framework (pytest) found for test." ; exit 126 ; fi +cleanup-cc-test-reporter: $(FETCH_CC_INCLUDE_PATH) + $(QUIET)$(CLEAN_CC_TOOL) || : + cleanup-dev-backups:: $(QUIET)$(RM) ./*/*~ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(RM) ./.*/*~ 2>$(ERROR_LOG_PATH) || : @@ -382,8 +386,7 @@ clean-docs: ./docs/ ./docs/Makefile ./docs/Makefile: ./docs/ $(QUIET)$(WAIT) ; -clean: clean-docs cleanup - $(QUIET)$(ECHO) "Cleaning Up." +clean: clean-docs cleanup cleanup-cc-test-reporter $(QUIET)$(COVERAGE) erase 2>$(ERROR_LOG_PATH) || true $(QUIET)$(RM) ./test-results/junit.xml 2>$(ERROR_LOG_PATH) || true $(QUIET)$(RM) ./MANIFEST.in 2>$(ERROR_LOG_PATH) || true diff --git a/includes/fetch-test-reporter b/includes/fetch-test-reporter index 87fdabd..d72e1f8 160000 --- a/includes/fetch-test-reporter +++ b/includes/fetch-test-reporter @@ -1 +1 @@ -Subproject commit 87fdabdae05040063a2cfef3f9275c5832f122e0 +Subproject commit d72e1f89868d8d6a170c847ae3b3b19866042ab4 From c1541d5743e295659be1de8bad4063d9ea959806 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 27 Mar 2025 22:51:58 -0700 Subject: [PATCH 40/58] [CONFIG] add submodule support to appveyor config (- WIP #79 -) --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 1d35fa4..41c157d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -25,6 +25,10 @@ init: choco install codecov || VER>NUL + git submodule sync || VER>NUL + + git submodule update --init || VER>NUL + dir build_script: - cmd: make clean From 6b599477ba3737a049f65719abf0e069dd444890 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 27 Mar 2025 23:21:26 -0700 Subject: [PATCH 41/58] [CONFIG] Improve Appveyor setup and install config and fix git submodule support (- WIP #79 -) --- .appveyor.yml | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 41c157d..37c16af 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,22 +9,20 @@ image: - Visual Studio 2022 - Visual Studio 2019 clone_depth: 50 -init: +install: + - cinst make + - cinst git + - cmd: >- + choco install python --pre || VER>NUL + + choco upgrade python --pre || VER>NUL + - cmd: >- + python -m pip install flake8 || VER>NUL + + python -m pip install coverage || VER>NUL + - cinst codecov +before_build: - cmd: >- - choco install make || VER>NUL - - choco install git || VER>NUL - - choco install python --pre || VER>NUL - - choco upgrade python --pre || VER>NUL - - python -m pip install flake8 || VER>NUL - - python -m pip install coverage || VER>NUL - - choco install codecov || VER>NUL - git submodule sync || VER>NUL git submodule update --init || VER>NUL From 8d5c97b563ea5a24c22c00abe64fa23f77c4dca5 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 27 Mar 2025 23:33:10 -0700 Subject: [PATCH 42/58] [CONFIG] Possible fix for regression (- WIP PR #82 -) --- .appveyor.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 37c16af..2aabac5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,8 +10,10 @@ image: - Visual Studio 2019 clone_depth: 50 install: - - cinst make - - cinst git + - cmd: >- + choco install make || VER>NUL + - cmd: >- + choco install git || VER>NUL - cmd: >- choco install python --pre || VER>NUL @@ -20,7 +22,8 @@ install: python -m pip install flake8 || VER>NUL python -m pip install coverage || VER>NUL - - cinst codecov + - cmd: >- + choco install codecov || VER>NUL before_build: - cmd: >- git submodule sync || VER>NUL From 01e170fcafe5c7963053d50655ed18f4226affbf Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 01:05:09 -0700 Subject: [PATCH 43/58] [CONFIG] Fix for old regression where COMMAND was not properly setup on UNIX. Does it still work on other platforms? (- WIP #79 -) --- Makefile | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 5ef1974..e618bce 100644 --- a/Makefile +++ b/Makefile @@ -30,17 +30,22 @@ ifeq "$(ERROR_LOG_PATH)" "" endif ifeq "$(COMMAND)" "" - COMMAND_CMD!=`command -v xcrun || command -v command || command which which || command -v which` - ifeq "$(COMMAND_CMD)" "*xcrun" + COMMAND_CMD=$(shell command -v xcrun || command -v command || command which which || command -v which) + COMMAND_TOOL=$(notdir $(COMMAND_CMD)) + ifeq "$(COMMAND_TOOL)" "xcrun" COMMAND_ARGS= --find endif - ifeq "$(COMMAND_CMD)" "*command" - COMMAND_ARGS= -pv + ifeq "$(COMMAND_TOOL)" "command" + COMMAND_ARGS= -v endif ifeq "$(COMMAND_CMD)" "" COMMAND_CMD="command" endif - COMMAND := $(COMMAND_CMD)$(COMMAND_ARGS) + ifdef COMMAND_ARGS + COMMAND := $(COMMAND_CMD) $(COMMAND_ARGS) + else + COMMAND := $(COMMAND_CMD) + endif endif ifeq "$(MAKE)" "" @@ -53,30 +58,32 @@ ifeq "$(ECHO)" "" ECHO=printf "%s\n" endif -ifdef "$(ACTION)" - SET_FILE_ATTR=$(COMMAND) xattr +ifdef ACTION + SET_FILE_ATTR=$(shell $(COMMAND) xattr) endif -ifdef "$(SET_FILE_ATTR)" +ifdef SET_FILE_ATTR CREATEDBYBUILDSYSTEM=-w com.apple.xcode.CreatedByBuildSystem true BSMARK=$(SET_FILE_ATTR) $(CREATEDBYBUILDSYSTEM) else - BSMARK=$(COMMAND) touch -a + BSMARK_CMD=$(shell $(COMMAND) touch) + BSMARK=$(BSMARK_CMD) -a endif ifeq "$(LINK)" "" - LINK=$(COMMAND) ln -sf + LINK_CMD=$(shell $(COMMAND) ln) + LINK=$(LINK_CMD) -sf endif # Python command configuration ifeq "$(PYTHON)" "" # Try to find python3, fallback to python - PY_CMD=$(COMMAND) python3 + PY_CMD=$(shell $(COMMAND) python3) ifneq "$(PY_CMD)" "" # Only use -B arg with python3 PY_ARGS=-B else - PY_CMD=$(COMMAND) python + PY_CMD=$(shell $(COMMAND) python) endif # Set PYTHON only if not already set PYTHON := $(PY_CMD) $(PY_ARGS) @@ -90,7 +97,7 @@ ifeq "$(COVERAGE)" "" endif # If COVERAGE is still not set, fall back to direct command ifeq "$(COVERAGE)" "" - COVERAGE!=$(COMMAND) coverage + COVERAGE=$(shell $(COMMAND) coverage) endif # Only set COV_CORE_* variables when COVERAGE is configured ifneq "$(COVERAGE)" "" @@ -121,7 +128,7 @@ ifeq "$(WAIT)" "" endif ifeq "$(INSTALL)" "" - INSTALL=install + INSTALL=$(shell $(COMMAND) install) ifeq "$(INST_OWN)" "" INST_OWN=-o root -g staff endif @@ -142,11 +149,12 @@ ifeq "$(LOG)" "no" endif ifeq "$(DO_FAIL)" "" - DO_FAIL=$(COMMAND) : + DO_FAIL=$(shell $(COMMAND) : ) endif ifeq "$(RM)" "" - RM=$(COMMAND) rm -f + RM_CMD=$(shell $(COMMAND) rm) + RM=$(RM_CMD) -f endif ifeq "$(RMDIR)" "" From 66c99dda6a0379a459f61d2c6370171ed7a23db2 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 14:27:36 -0700 Subject: [PATCH 44/58] [PATCH] Fixes for some regressions (- WIP #82 -) --- Makefile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index e618bce..1a84b71 100644 --- a/Makefile +++ b/Makefile @@ -253,19 +253,19 @@ purge: legacy-purge $(QUIET)$(ECHO) "$@: Done." test-reports: .env - $(QUIET)mkdir $(INST_OPTS) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; - $(QUIET)$(BSMARK) ./test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)mkdir $(INST_OPTS) $(dir $(abspath $(lastword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)$(BSMARK) $(dir $(abspath $(lastword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(ECHO) "$@: Done." -test-reqs: cc-test-reporter test-reports init - $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r tests-requirements.txt 2>$(ERROR_LOG_PATH) || true +test-reqs: .env init cc-test-reporter test-reports + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r test-requirements.txt 2>$(ERROR_LOG_PATH) || true -just-test: cleanup cc-test-reporter .env - $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s ./tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s ./tests -t ./ || DO_FAIL="exit 2" ; +just-test: cleanup + $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s $(dir $(abspath $(lastword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s $(dir $(abspath $(lastword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; -test: just-test +test: test-reqs just-test $(QUIET)$(DO_FAIL) ; $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; @@ -276,8 +276,8 @@ test-tox: cleanup $(QUIET)tox -v -- || tail -n 500 .tox/py*/log/py*.log 2>/dev/null $(QUIET)$(ECHO) "$@: Done." -test-pytest: cleanup MANIFEST.in cc-test-reporter must_have_pytest test-reports - $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=. || DO_FAIL="exit 2" ; +test-pytest: cleanup MANIFEST.in test-reqs must_have_pytest test-reports + $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(CA_TOOL) $(CA_TOOL_ARGS) || : ; $(QUIET)$(DS_TOOL) $(DS_TOOL_ARGS) || : ; From c08d48b12ead132d76e7f1babcf870ba78e0afb0 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 14:54:13 -0700 Subject: [PATCH 45/58] [PATCH] More cleanup and regression fixes (- WIP #82 -) --- .coveragerc | 36 ++++++++++++++++++++++++++++++------ Makefile | 10 +++++----- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.coveragerc b/.coveragerc index 59ae610..3b34542 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,24 +1,48 @@ [run] +concurrency = multiprocessing parallel = True +sigterm = True +# enable if you want to consider branches in coverage +# branch = True [report] -include = pythonrepo*,tests* +include = pythonrepo/*,tests/* # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma pragma: no cover + pass + except ImportError + except ModuleNotFoundError except Exception - except BaseException: + except BaseException + except UnicodeDecodeError # Don't complain if tests don't hit defensive assertion code: raise AssertionError raise NotImplementedError raise ImportError + raise ModuleNotFoundError except unittest.SkipTest - except IOError - except OSError - + except ..Error # Don't complain if non-runnable code isn't run: - if __name__ in '__main__': + if __name__ .. .__main__.: + if __name__ in u'__main__': + if __name__ in u"__main__": + os.abort() + exit ignore_errors = True +partial_branches = + # Have to re-enable the standard pragma rules + pragma: no branch + skipcq: PYL- + finally: + @ + except unittest.SkipTest + self.skipTest + self.fail + # Don't complain if non-runnable code isn't run: + if __name__ in u'__main__': + if __name__ in u"__main__": + if __name__ in '__main__': diff --git a/Makefile b/Makefile index 1a84b71..f1bf5d3 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ ifeq "$(COVERAGE)" "" # Only set COV_CORE_* variables when COVERAGE is configured ifneq "$(COVERAGE)" "" #COV_CORE_SOURCE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/ - COV_CORE_CONFIG = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.coveragerc + COV_CORE_CONFIG = $(dir $(abspath $(firstword $(MAKEFILE_LIST)))).coveragerc COV_CORE_DATAFILE = .coverage endif endif @@ -253,15 +253,15 @@ purge: legacy-purge $(QUIET)$(ECHO) "$@: Done." test-reports: .env - $(QUIET)mkdir $(INST_OPTS) $(dir $(abspath $(lastword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; - $(QUIET)$(BSMARK) $(dir $(abspath $(lastword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)mkdir $(INST_OPTS) $(dir $(abspath $(firstword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)$(BSMARK) $(dir $(abspath $(firstword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(ECHO) "$@: Done." test-reqs: .env init cc-test-reporter test-reports $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r test-requirements.txt 2>$(ERROR_LOG_PATH) || true just-test: cleanup - $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s $(dir $(abspath $(lastword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s $(dir $(abspath $(lastword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; + $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s $(dir $(abspath $(firstword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s $(dir $(abspath $(firstword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; $(QUIET)$(WAIT) ; $(QUIET)$(DO_FAIL) ; @@ -277,7 +277,7 @@ test-tox: cleanup $(QUIET)$(ECHO) "$@: Done." test-pytest: cleanup MANIFEST.in test-reqs must_have_pytest test-reports - $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; + $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=$(dir $(abspath $(firstword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(CA_TOOL) $(CA_TOOL_ARGS) || : ; $(QUIET)$(DS_TOOL) $(DS_TOOL_ARGS) || : ; From 192fc5dae6ee074dfd7102c47c8ad23921fc1a4e Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 15:16:04 -0700 Subject: [PATCH 46/58] [PATCH] Part 1 of many - possible fix for GHA cross-python testing (- WIP #79 -) --- .github/workflows/Tests.yml | 9 +++------ Makefile | 8 ++++++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 01ab20f..0a8db2d 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -202,10 +202,7 @@ jobs: if: ${{ !cancelled() && runner.os == 'Windows' }} run: python -m pip install --upgrade pip - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} - run: | - pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; - pip install -r ./requirements.txt ; - pip install -r ./test-requirements.txt || true ; + run: make -f Makefile test-reqs ; - name: Pre-Clean id: clean run: make -j1 -f Makefile clean || true ; @@ -241,10 +238,10 @@ jobs: run: | ./cc-test-reporter after-build --exit-code 0 || true ; - name: deepsource for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash run: | - if [ "$OS" == "ubuntu-latest" ] ; then ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; else echo "SKIP deepsource" ; fi ; + ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; diff --git a/Makefile b/Makefile index f1bf5d3..eca5f80 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,12 @@ endif # Python command configuration ifeq "$(PYTHON)" "" - # Try to find python3, fallback to python - PY_CMD=$(shell $(COMMAND) python3) + ifneq "$(PYTHON_VERSION)" "" + PY_CMD=$(shell $(COMMAND) python$(PYTHON_VERSION)) + else + # Try to find python3, fallback to python + PY_CMD=$(shell $(COMMAND) python3) + endif ifneq "$(PY_CMD)" "" # Only use -B arg with python3 PY_ARGS=-B From de98224b885db9500933e3dc070d8babeb4c50dc Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 16:11:00 -0700 Subject: [PATCH 47/58] [PATCH] Part 2 of many - Possible fix for windows trouble in GHA (- WIP PR #82 -) --- .github/workflows/Tests.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 0a8db2d..61174b4 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -173,6 +173,9 @@ jobs: if: ${{ success() }} needs: [BUILD, MATS] runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash timeout-minutes: 10 strategy: matrix: @@ -202,9 +205,11 @@ jobs: if: ${{ !cancelled() && runner.os == 'Windows' }} run: python -m pip install --upgrade pip - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} + shell: bash run: make -f Makefile test-reqs ; - name: Pre-Clean id: clean + shell: bash run: make -j1 -f Makefile clean || true ; - name: Install code-climate tools for ${{ matrix.python-version }} if: ${{ !cancelled() && runner.os != 'Windows' }} @@ -212,9 +217,11 @@ jobs: run: ./tests/fetch-test-reporter || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} if: ${{ matrix.python-version == '3.9' }} + shell: bash run: make -f Makefile test ; - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} if: ${{ matrix.python-version != '3.9' }} + shell: bash run: make -f Makefile test-pytest ; - name: Upload Python ${{ matrix.python-version }} coverage to Codecov uses: codecov/codecov-action@v5 @@ -244,6 +251,7 @@ jobs: ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; - name: Post-Clean id: post + shell: bash run: make -j1 -f Makefile clean || true ; if: ${{ !cancelled() }} @@ -267,10 +275,8 @@ jobs: with: python-version: "3.12" - name: Install dependencies for python Linters - run: | - pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; - pip install -r ./requirements.txt ; - pip install -r ./test-requirements.txt || true ; + shell: bash + run: make -f Makefile test-reqs || true ; - name: Pre-Clean id: clean run: make -j1 -f Makefile clean || true ; @@ -314,18 +320,18 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} - run: | - pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" ; - pip install -r ./requirements.txt ; - pip install -r ./test-requirements.txt || true ; + shell: bash + run: make -f Makefile test-reqs ; - name: Install code-climate tools for ${{ matrix.python-version }} on ${{ matrix.os }} if: ${{ !cancelled() && runner.os != 'Windows' }} run: if [ $OS == macos-latest ] ; then ./tests/fetch-test-reporter || true ; fi ; shell: bash - name: Pre-Clean id: clean-prep + shell: bash run: make -j1 -f Makefile clean ; - name: Pre-build for Python ${{ matrix.python-version }} on ${{ matrix.os }} + shell: bash run: make -j1 -f Makefile build ; if: ${{ success() }} - name: Pre-install for Python ${{ matrix.python-version }} on ${{ matrix.os }} From d71e4fe8c194eecc30dab0c75c0fbb31daeb1925 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 17:05:54 -0700 Subject: [PATCH 48/58] [PATCH] part 3 of many - possible improvement for windows GHA runners (- WIP PR #82 -) --- .github/workflows/Tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 61174b4..e8084a7 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -204,6 +204,10 @@ jobs: - name: Fix braindead windows ${{ matrix.python-version }} on ${{ matrix.os }} if: ${{ !cancelled() && runner.os == 'Windows' }} run: python -m pip install --upgrade pip + - name: Add Git Unix tools to PATH for braindead windows ${{ matrix.os }} + if: ${{ !cancelled() && runner.os == 'Windows' }} + run: | + echo "C:\Program Files\Git\usr\bin" >> $env:GITHUB_PATH - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} shell: bash run: make -f Makefile test-reqs ; From bcba1a193d62d8d3052363634b67bf18ff6dacc3 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 21:20:29 -0700 Subject: [PATCH 49/58] [PATCH] Small refactor to handle windows with non-functioning `command` command (- WIP PR #82 -) --- .github/workflows/Tests.yml | 4 ++++ Makefile | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index e8084a7..2b13feb 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -208,6 +208,10 @@ jobs: if: ${{ !cancelled() && runner.os == 'Windows' }} run: | echo "C:\Program Files\Git\usr\bin" >> $env:GITHUB_PATH + - name: VERIFY Git Unix tools in PATH for braindead windows ${{ matrix.os }} + if: ${{ !cancelled() && runner.os == 'Windows' }} + run: | + command -v touch - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} shell: bash run: make -f Makefile test-reqs ; diff --git a/Makefile b/Makefile index eca5f80..30d4e97 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,12 @@ ifeq "$(COMMAND)" "" endif endif +# Check if POSIX touch command can be resolved by COMMAND in the runtime env +ifeq "$(notdir $(shell $(COMMAND) touch))" "" + # This is a non-POSIX environment, so try Windows fallback + COMMAND := where +endif + ifeq "$(MAKE)" "" # just no cmake please MAKEFLAGS=$(MAKEFLAGS) -s @@ -72,7 +78,11 @@ endif ifeq "$(LINK)" "" LINK_CMD=$(shell $(COMMAND) ln) - LINK=$(LINK_CMD) -sf + ifneq "$(LINK_CMD)" "" + LINK=$(LINK_CMD) -sf + else + LINK=$(ECHO) "::debug:: Linking is not supported for file: " + endif endif # Python command configuration @@ -90,7 +100,11 @@ ifeq "$(PYTHON)" "" PY_CMD=$(shell $(COMMAND) python) endif # Set PYTHON only if not already set - PYTHON := $(PY_CMD) $(PY_ARGS) + ifdef PY_ARGS + PYTHON := $(PY_CMD) $(PY_ARGS) + else + PYTHON := $(PY_CMD) + endif endif # Coverage configuration From 8b240f45bae86c34398788ac0a4c6e5e79bf0ece Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 22:14:39 -0700 Subject: [PATCH 50/58] [PATCH] BROKEN just testing behavior --- .github/workflows/Tests.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 2b13feb..d0552bd 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -208,10 +208,17 @@ jobs: if: ${{ !cancelled() && runner.os == 'Windows' }} run: | echo "C:\Program Files\Git\usr\bin" >> $env:GITHUB_PATH - - name: VERIFY Git Unix tools in PATH for braindead windows ${{ matrix.os }} + - name: VERIFY that PATH resolution works somehow for braindead windows on ${{ matrix.os }} if: ${{ !cancelled() && runner.os == 'Windows' }} run: | - command -v touch + command -v touch ; + command -v python ; + command -v python3 ; + command -v make ; + where touch ; + where python ; + where python3 ; + where make ; - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} shell: bash run: make -f Makefile test-reqs ; From 1954399ab3e53ff0bbd85b5ab137ed1e85d443ea Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 22:41:55 -0700 Subject: [PATCH 51/58] [BROKEN] do not use --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 30d4e97..3feba4a 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,8 @@ endif # Check if POSIX touch command can be resolved by COMMAND in the runtime env ifeq "$(notdir $(shell $(COMMAND) touch))" "" - # This is a non-POSIX environment, so try Windows fallback - COMMAND := where + # This is a non-POSIX environment, so try Windows fallback logic + COMMAND!=`command -v` endif ifeq "$(MAKE)" "" From d376b9ee07c188c0add93a4d290eac5cb2debd0c Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 23:08:05 -0700 Subject: [PATCH 52/58] [PATCH] Also Broken, just a test. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3feba4a..54047e8 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ endif # Check if POSIX touch command can be resolved by COMMAND in the runtime env ifeq "$(notdir $(shell $(COMMAND) touch))" "" # This is a non-POSIX environment, so try Windows fallback logic - COMMAND!=`command -v` + COMMAND="which" endif ifeq "$(MAKE)" "" From 9b2efdbd0c1df4f133447bc31fd38e92147dc240 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Fri, 28 Mar 2025 23:57:19 -0700 Subject: [PATCH 53/58] [PATCH] Possible fix for regressions (- WIP PR #82 -) --- .github/workflows/Tests.yml | 15 --------------- Makefile | 1 + 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index d0552bd..61174b4 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -204,21 +204,6 @@ jobs: - name: Fix braindead windows ${{ matrix.python-version }} on ${{ matrix.os }} if: ${{ !cancelled() && runner.os == 'Windows' }} run: python -m pip install --upgrade pip - - name: Add Git Unix tools to PATH for braindead windows ${{ matrix.os }} - if: ${{ !cancelled() && runner.os == 'Windows' }} - run: | - echo "C:\Program Files\Git\usr\bin" >> $env:GITHUB_PATH - - name: VERIFY that PATH resolution works somehow for braindead windows on ${{ matrix.os }} - if: ${{ !cancelled() && runner.os == 'Windows' }} - run: | - command -v touch ; - command -v python ; - command -v python3 ; - command -v make ; - where touch ; - where python ; - where python3 ; - where make ; - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} shell: bash run: make -f Makefile test-reqs ; diff --git a/Makefile b/Makefile index 54047e8..b9678dd 100644 --- a/Makefile +++ b/Makefile @@ -195,6 +195,7 @@ include $(FETCH_CC_INCLUDE_PATH) $(QUIET)$(ECHO) " ECHO: '$(ECHO)'" $(QUIET)$(ECHO) " PYTHON: '$(PYTHON)'" $(QUIET)$(ECHO) " COVERAGE: '$(COVERAGE)'" + $(QUIET)$(ECHO) " BSMARK: '$(BSMARK)'" $(QUIET)$(ECHO) " FETCH_CC_INCLUDE_PATH: '$(FETCH_CC_INCLUDE_PATH)'" $(QUIET)$(ECHO) " FETCH_CC_TOOL: '$(FETCH_CC_TOOL)'" $(QUIET)$(ECHO) " CC_TOOL: '$(CC_TOOL)'" From abda4c329655e2dec41a411dc596278b77deab72 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Sat, 29 Mar 2025 01:31:39 -0700 Subject: [PATCH 54/58] [PATCH] Possible regression fix for coverage (- WIP PR #82 -) --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 3b34542..8d105da 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,7 +6,7 @@ sigterm = True # branch = True [report] -include = pythonrepo/*,tests/* +include = pythonrepo/*,tests* # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma From 4c950d8192904685a075b399b457bac9b6bf8b77 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Sat, 29 Mar 2025 01:34:33 -0700 Subject: [PATCH 55/58] [PATCH] Another Possible fix for coverage (- WIP PR #82 -) --- .coveragerc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index 8d105da..9d7043d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,11 +2,10 @@ concurrency = multiprocessing parallel = True sigterm = True -# enable if you want to consider branches in coverage -# branch = True +branch = True [report] -include = pythonrepo/*,tests* +include = pythonrepo/*,tests/* # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma From 46fc30b35392df42fbe990c4cd5abb50e4c77ce5 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Sat, 29 Mar 2025 01:34:33 -0700 Subject: [PATCH 56/58] [PATCH] Another Possible fix for coverage (- WIP PR #82 -) --- pythonrepo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonrepo/__init__.py b/pythonrepo/__init__.py index b24afa4..40154f0 100644 --- a/pythonrepo/__init__.py +++ b/pythonrepo/__init__.py @@ -30,7 +30,7 @@ try: import sys import os - if str(__module__) in __file__: + if str(__module__) in __file__: # pragma: no branch __parentPath = os.path.join( os.path.dirname(__file__), '..' ) From 24035af0b2db5d022ebe406e968be7c3e0059c31 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Sat, 29 Mar 2025 01:58:35 -0700 Subject: [PATCH 57/58] [PATCH] Coverage fixup for #82 --- .codecov.yml | 6 +++++- .coveragerc | 13 +++++++++++-- Makefile | 3 ++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index 8ace9c0..aa64926 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -39,6 +39,10 @@ coverage: github_checks: annotations: true flags: + pythonrepo: + paths: + - "pythonrepo/" tests: paths: - - tests \ No newline at end of file + - "tests/" + - "!pythonrepo/" diff --git a/.coveragerc b/.coveragerc index 9d7043d..a8853b9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,10 +2,11 @@ concurrency = multiprocessing parallel = True sigterm = True -branch = True +# enable if you want to consider branches in coverage +# branch = True [report] -include = pythonrepo/*,tests/* +include = pythonrepo/* # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma @@ -45,3 +46,11 @@ partial_branches = if __name__ in u'__main__': if __name__ in u"__main__": if __name__ in '__main__': + if __sys_path__ not in sys.path: + # don't complain about sys.modules + sys.modules + not in sys.modules: + if context.__name__ is None: + if 'os' not in sys.modules: + if 'os.path' not in sys.modules: + if 'argparse' not in sys.modules: diff --git a/Makefile b/Makefile index b9678dd..fa0bd30 100644 --- a/Makefile +++ b/Makefile @@ -287,7 +287,7 @@ just-test: cleanup test: test-reqs just-test $(QUIET)$(DO_FAIL) ; $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; - $(QUIET)$(COVERAGE) report -m --include=* 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(COVERAGE) report -m --include=pythonrepo/* 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(ECHO) "$@: Done." @@ -384,6 +384,7 @@ cleanup-src-dir: cleanup-dev-backups cleanup-mac-dir-store cleanup: cleanup-tests cleanup-pythonrepo cleanup-pythonrepo-eggs cleanup-src-dir $(QUIET)$(RM) ./.coverage 2>$(ERROR_LOG_PATH) || true $(QUIET)$(RM) ./coverage*.xml 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./.coverage.* 2>$(ERROR_LOG_PATH) || true $(QUIET)$(RM) ./sitecustomize.py 2>$(ERROR_LOG_PATH) || true $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || true $(QUIET)$(WAIT) ; From 6f3ed580a7b8abe6d863bc95042b0edd42ea25ad Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Wed, 2 Apr 2025 00:53:10 -0700 Subject: [PATCH 58/58] Update Makefile (- WIP PR #82 -) Signed-off-by: Mr. Walls --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fa0bd30..588c7bf 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ endif # Check if POSIX touch command can be resolved by COMMAND in the runtime env ifeq "$(notdir $(shell $(COMMAND) touch))" "" - # This is a non-POSIX environment, so try Windows fallback logic + # This is a non-POSIX environment, so try GHA-Windows-Latest Specific fallback logic COMMAND="which" endif