diff --git a/.appveyor.yml b/.appveyor.yml index 1d35fa4..2aabac5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,21 +9,26 @@ image: - Visual Studio 2022 - Visual Studio 2019 clone_depth: 50 -init: +install: + - cmd: >- + choco install make || VER>NUL + - cmd: >- + choco install git || VER>NUL + - cmd: >- + choco install python --pre || VER>NUL + + choco upgrade python --pre || VER>NUL + - cmd: >- + python -m pip install flake8 || VER>NUL + + python -m pip install coverage || VER>NUL + - cmd: >- + choco install codecov || VER>NUL +before_build: - cmd: >- - choco install make || VER>NUL + git submodule sync || VER>NUL - choco install git || VER>NUL - - choco install python --pre || VER>NUL - - choco upgrade python --pre || VER>NUL - - python -m pip install flake8 || VER>NUL - - python -m pip install coverage || VER>NUL - - choco install codecov || VER>NUL + git submodule update --init || VER>NUL dir build_script: diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e22ae8..144e304 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,37 +1,67 @@ -version: 2 +--- +version: 2.1 +commands: + cleanup: + steps: + - run: + shell: /bin/bash + name: "Cleanup" + command: | + make clean + when: always + reposync: + steps: + - run: + shell: /bin/bash + name: "Fetch" + command: | + git fetch --all ; + when: always + - run: + shell: /bin/bash + name: "Sync Submodules" + command: | + git submodule sync ; + when: on_success + - run: + shell: /bin/bash + name: "Update Submodules" + command: | + git submodule update --init ; + when: on_success + +parameters: + python-version: + type: string + default: "3.12" + jobs: build: docker: - - image: cimg/python:3.7 - - image: cimg/python:3.8 - - image: cimg/python:3.9 - - image: cimg/python:3.10 - - image: cimg/python:3.11 - - image: cimg/python:3.12 + - image: cimg/python:<< pipeline.parameters.python-version >> + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 SHELL: /bin/bash working_directory: ~/python-repo steps: - checkout - - run: - name: "fetch and pull" - command: | - git fetch && git pull --all || true + - reposync - run: shell: /bin/bash name: "install depends attempt" command: | - python3 -m pip install --user -r ./requirements.txt || true + python3 -m pip install --user -r ./requirements.txt || : ; when: on_success - run: shell: /bin/bash - name: "install test extras attempt" + name: "install test depends attempt" command: | - python3 -m pip install --user -r ./test-requirements.txt || true + python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; + when: on_success - save_cache: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} paths: @@ -39,18 +69,14 @@ jobs: test: docker: - - image: cimg/python:3.7 - - image: cimg/python:3.8 - - image: cimg/python:3.9 - - image: cimg/python:3.10 - - image: cimg/python:3.11 - - image: cimg/python:3.12 + - image: cimg/python:<< pipeline.parameters.python-version >> parallelism: 2 + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 SHELL: /bin/bash working_directory: ~/python-repo steps: @@ -58,43 +84,29 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - run: shell: /bin/bash - name: "clean up for test" + name: "Installing deps for test" command: | - make clean - when: always + python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; + when: on_success + - cleanup - run: shell: /bin/bash name: "Unit Tests" command: | make test when: on_success - - run: - shell: /bin/bash - name: "clean up from test" - command: | - make clean - when: always - - run: - shell: /bin/bash - name: "clean up from FAIL" - command: | - make clean - when: on_fail + - cleanup pytest: docker: - - image: cimg/python:3.7 - - image: cimg/python:3.8 - - image: cimg/python:3.9 - - image: cimg/python:3.10 - - image: cimg/python:3.11 - - image: cimg/python:3.12 + - image: cimg/python:<< pipeline.parameters.python-version >> parallelism: 2 + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 SHELL: /bin/bash working_directory: ~/python-repo steps: @@ -102,16 +114,17 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - run: shell: /bin/bash - name: "setup up for pytest" + name: "set up depends" command: | - python3 -m pip install --upgrade --user -r ./test-requirements.txt || true + python3 -m pip install --upgrade --user -r ./requirements.txt || : ; when: on_success - run: shell: /bin/bash - name: "clean up for pytest" + name: "install test-reqs attempt" command: | - make clean - when: always + python3 -m pip install --upgrade --user -r ./test-requirements.txt || : ; + when: on_success + - cleanup - run: shell: /bin/bash name: "pytest Unit Tests" @@ -124,55 +137,41 @@ jobs: - store_artifacts: path: test-reports when: on_success - - run: - shell: /bin/bash - name: "clean up from pytest" - command: | - make clean - when: always - - run: - shell: /bin/bash - name: "clean up from FAIL" - command: | - make clean - when: on_fail + - cleanup lint: docker: - - image: cimg/python:3.11 + - image: cimg/python:<< pipeline.parameters.python-version >> + resource_class: medium environment: CI: cicleci DEBIAN_FRONTEND: noninteractive LANG: en_US.UTF-8 SHELL: /bin/bash - LC_CTYPE: en_EN.UTF-8 + LC_CTYPE: en_US.UTF-8 working_directory: ~/python-repo steps: - restore_cache: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - run: - name: "install linters extras attempt" + name: "install reqs attempt" command: | - python3 -m pip install --upgrade --user -r ./test-requirements.txt || true + python3 -m pip install --user -r ./requirements.txt || : ; - run: - shell: /bin/bash - name: "clean up for test" + name: "install test-reqs attempt" command: | - make clean + python3 -m pip install --user -r ./test-requirements.txt || : ; + - cleanup - run: shell: /bin/bash name: "check code style and spelling" command: | - make test-style || python3 -m flake8 --ignore=W191,W391,E117 --max-line-length=100 --verbose --count --config=.flake8.ini --max-complexity=10 - - run: - shell: /bin/bash - name: "clean up when done" - command: | - make clean + make test-style || python3 -m flake8 --verbose --count --config=.flake8.ini + - cleanup workflows: version: 2 - workflow: + test-matrix: jobs: - build - test: @@ -184,3 +183,4 @@ workflows: - pytest: requires: - build + - test diff --git a/.codecov.yml b/.codecov.yml index 8ace9c0..aa64926 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -39,6 +39,10 @@ coverage: github_checks: annotations: true flags: + pythonrepo: + paths: + - "pythonrepo/" tests: paths: - - tests \ No newline at end of file + - "tests/" + - "!pythonrepo/" diff --git a/.coveragerc b/.coveragerc index 59ae610..a8853b9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,24 +1,56 @@ [run] +concurrency = multiprocessing parallel = True +sigterm = True +# enable if you want to consider branches in coverage +# branch = True [report] -include = pythonrepo*,tests* +include = pythonrepo/* # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma pragma: no cover + pass + except ImportError + except ModuleNotFoundError except Exception - except BaseException: + except BaseException + except UnicodeDecodeError # Don't complain if tests don't hit defensive assertion code: raise AssertionError raise NotImplementedError raise ImportError + raise ModuleNotFoundError except unittest.SkipTest - except IOError - except OSError - + except ..Error # Don't complain if non-runnable code isn't run: - if __name__ in '__main__': + if __name__ .. .__main__.: + if __name__ in u'__main__': + if __name__ in u"__main__": + os.abort() + exit ignore_errors = True +partial_branches = + # Have to re-enable the standard pragma rules + pragma: no branch + skipcq: PYL- + finally: + @ + except unittest.SkipTest + self.skipTest + self.fail + # Don't complain if non-runnable code isn't run: + if __name__ in u'__main__': + if __name__ in u"__main__": + if __name__ in '__main__': + if __sys_path__ not in sys.path: + # don't complain about sys.modules + sys.modules + not in sys.modules: + if context.__name__ is None: + if 'os' not in sys.modules: + if 'os.path' not in sys.modules: + if 'argparse' not in sys.modules: diff --git a/.flake8.ini b/.flake8.ini index 6d4e752..5337f6c 100644 --- a/.flake8.ini +++ b/.flake8.ini @@ -1,13 +1,14 @@ [flake8] -select = C,E,F,W,B,B950 +select = C,D,E,F,W,B,B950 # Ignore specific warnings and errors according to CEP-8 style -extend-ignore = - W191, # Indentation contains tabs - W391, # Blank line at end of file - E117, # Over-indented - D208, # Docstring is over-indented - D203, # 1 blank line required before class docstring - CEP-7 - D212, # Multi-line docstring summary should start at the first line - CEP-7 +extend-ignore = E117,D203,D208,D212,W191,W391 +# CEP-8 Custom Exceptions: +# W191, # Indentation contains tabs +# W391, # Blank line at end of file +# E117, # Over-indented +# D208, # Docstring is over-indented +# D203, # 1 blank line required before class docstring - CEP-7 +# D212, # Multi-line docstring summary should start at the first line - CEP-7 # Ignore long lines as specified in CEP-8 max-line-length = 100 extend-exclude = diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 16a0104..61174b4 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -4,6 +4,8 @@ on: branches: - master - stable + - patch* + - feature* tags: - v* pull_request: @@ -11,7 +13,6 @@ on: - opened - edited - reopened - - synchronize - ready_for_review # Declare default permissions as read only. @@ -28,6 +29,9 @@ jobs: LANG: "en_US.UTF-8" steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.12" @@ -43,7 +47,7 @@ jobs: BOOTSTRAP: - if: ${{ always() }} + if: ${{ !cancelled() }} needs: BUILD runs-on: ubuntu-latest defaults: @@ -54,22 +58,10 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] lang-var: ["de.utf-8", "jp.utf-8"] experimental: [true] include: - - python-version: 3.7 - lang-var: "en_US.utf-8" - experimental: false - - python-version: 3.8 - lang-var: "en_US.utf-8" - experimental: false - - python-version: 3.9 - lang-var: "de.utf-8" - experimental: false - - python-version: 3.9 - lang-var: "jp.utf-8" - experimental: false - python-version: 3.9 lang-var: "en_US.utf-8" experimental: false @@ -87,13 +79,16 @@ jobs: LANG: ${{ matrix.lang-var }} steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Setup dependencies run: | - python -m pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" + python -m pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" pip install -r ./requirements.txt pip install -r ./test-requirements.txt || true - name: Pre-build @@ -126,12 +121,12 @@ jobs: run: | make -j1 -f Makefile purge || true ; make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} shell: bash MATS: - if: ${{ always() }} + if: ${{ !cancelled() }} needs: BUILD runs-on: ubuntu-latest defaults: @@ -140,12 +135,15 @@ jobs: timeout-minutes: 10 strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: ["3.9.21", "3.10", "3.11", "3.12"] env: PYTHON_VERSION: ${{ matrix.python-version }} LANG: "en_US.utf-8" steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -169,18 +167,20 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} - + if: ${{ !cancelled() }} - COVERAGE-MATS: + COVERAGE: if: ${{ success() }} needs: [BUILD, MATS] runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash timeout-minutes: 10 strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] env: OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} @@ -188,116 +188,40 @@ jobs: COVERAGE_RCFILE: ./.coveragerc COV_CORE_SOURCE: ./ COV_CORE_CONFIG: ./.coveragerc - COV_CORE_DATAFILE: ./coverage.xml + COV_CORE_DATAFILE: ./coverage.xml\ + DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }} CODECLIMATE_REPO_TOKEN: ${{ secrets.CODECLIMATE_TOKEN }} CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Fix braindead windows ${{ matrix.python-version }} on ${{ matrix.os }} + if: ${{ !cancelled() && runner.os == 'Windows' }} + run: python -m pip install --upgrade pip - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} - run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" - pip install -r ./requirements.txt - pip install -r ./test-requirements.txt || true - - name: Install code-climate tools for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" - shell: bash - run: if [ "$OS" == "macos-latest" ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; - - name: Install deepsource tools for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" shell: bash - run: | - if [ "$OS" == "ubuntu-latest" ] ; then (curl https://deepsource.io/cli | sh) || true ; else echo "SKIP deepsource" ; fi ; + run: make -f Makefile test-reqs ; - name: Pre-Clean id: clean - run: make -j1 -f Makefile clean || true ; - - name: Generate Coverage for py3.9 on ${{ matrix.os }} - if: ${{ runner.python-version }} == "3.9" - run: make -f Makefile test ; - - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} - if: ${{ runner.python-version }} != "3.9" - run: make -f Makefile test-pytest ; - - name: Upload Python ${{ matrix.python-version }} coverage to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - directory: . - flags: ${{ matrix.os }},${{ matrix.python-version }} - name: pythonrepo-github-${{ matrix.os }}-${{ matrix.python-version }} - verbose: true - fail_ci_if_error: false - - name: Upload Python ${{ matrix.python-version }} Artifact - uses: actions/upload-artifact@v4 - with: - name: Test-Report-${{ matrix.os }}-${{ matrix.python-version }} - path: ./test-reports/ - if-no-files-found: ignore - - name: code-climate for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" - shell: bash - run: | - if [ "$OS" == "macos-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; - - name: deepsource for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" shell: bash - run: | - if [ "$OS" == "ubuntu-latest" ] ; then ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; else echo "SKIP deepsource" ; fi ; - - name: Post-Clean - id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} - - COVERAGE: - if: ${{ success() }} - needs: [BUILD, MATS, COVERAGE-MATS] - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - strategy: - matrix: - os: [ubuntu-latest, macos-13, windows-latest] - python-version: [3.7, 3.8, 3.9, "3.10"] - env: - OS: ${{ matrix.os }} - PYTHON_VERSION: ${{ matrix.python-version }} - LANG: "en_US.utf-8" - COVERAGE_RCFILE: ./.coveragerc - COV_CORE_SOURCE: ./ - COV_CORE_CONFIG: ./.coveragerc - COV_CORE_DATAFILE: ./coverage.xml - CODECLIMATE_REPO_TOKEN: ${{ secrets.CODECLIMATE_TOKEN }} - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - steps: - - uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} - run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" ; - pip install -r ./requirements.txt ; - pip install -r ./test-requirements.txt || true ; - name: Install code-climate tools for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" - shell: bash - run: if [ "$OS" == "ubuntu-latest" ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; - - name: Install deepsource tools for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash - run: | - if [ "$OS" == "ubuntu-latest" ] ; then (curl https://deepsource.io/cli | sh) || true ; else echo "SKIP deepsource" ; fi ; - - name: Pre-Clean - id: clean - run: make -j1 -f Makefile clean || true ; + run: ./tests/fetch-test-reporter || true ; - name: Generate Coverage for py3.9 on ${{ matrix.os }} - if: ${{ runner.python-version }} == "3.9" + if: ${{ matrix.python-version == '3.9' }} + shell: bash run: make -f Makefile test ; - name: Generate Coverage for py${{ matrix.python-version }} on ${{ matrix.os }} - if: ${{ runner.python-version }} != "3.9" + if: ${{ matrix.python-version != '3.9' }} + shell: bash run: make -f Makefile test-pytest ; - name: Upload Python ${{ matrix.python-version }} coverage to Codecov uses: codecov/codecov-action@v5 @@ -316,19 +240,20 @@ jobs: path: ./test-reports/ if-no-files-found: ignore - name: code-climate for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash run: | - if [ "$OS" == "ubuntu-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; + ./cc-test-reporter after-build --exit-code 0 || true ; - name: deepsource for ${{ matrix.python-version }} - if: ${{ runner.os }} == "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} shell: bash run: | - if [ "$OS" == "ubuntu-latest" ] ; then ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; else echo "SKIP deepsource" ; fi ; + ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml 2>/dev/null || true ; - name: Post-Clean id: post + shell: bash run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} STYLE: if: ${{ success() }} @@ -337,20 +262,21 @@ jobs: timeout-minutes: 10 env: - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.12' LANG: "en_US.utf-8" steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies for python Linters - run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" ; - pip install -r ./requirements.txt ; - pip install -r ./test-requirements.txt || true ; + shell: bash + run: make -f Makefile test-reqs || true ; - name: Pre-Clean id: clean run: make -j1 -f Makefile clean || true ; @@ -361,7 +287,7 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} INTEGRATION: @@ -386,23 +312,26 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies for python ${{ matrix.python-version }} on ${{ matrix.os }} - run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" ; - pip install -r ./requirements.txt ; - pip install -r ./test-requirements.txt || true ; - - name: Install code-climate tools for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" - run: if [ $OS == macos-latest ] ; then ./tests/fetch_cc-test-reporter || true ; fi ; + shell: bash + run: make -f Makefile test-reqs ; + - name: Install code-climate tools for ${{ matrix.python-version }} on ${{ matrix.os }} + if: ${{ !cancelled() && runner.os != 'Windows' }} + run: if [ $OS == macos-latest ] ; then ./tests/fetch-test-reporter || true ; fi ; shell: bash - name: Pre-Clean id: clean-prep + shell: bash run: make -j1 -f Makefile clean ; - name: Pre-build for Python ${{ matrix.python-version }} on ${{ matrix.os }} + shell: bash run: make -j1 -f Makefile build ; if: ${{ success() }} - name: Pre-install for Python ${{ matrix.python-version }} on ${{ matrix.os }} @@ -430,17 +359,17 @@ jobs: path: ./test-reports/ if-no-files-found: ignore - name: code-climate for ${{ matrix.python-version }} - if: ${{ runner.os }} != "Linux" + if: ${{ !cancelled() && runner.os != 'Windows' }} run: | - if [ "$OS" == "macos-latest" ] ; then ./cc-test-reporter after-build --exit-code 0 || true ; else echo "SKIP code climate" ; fi ; + ./cc-test-reporter after-build --exit-code 0 || true ; - name: Post-purge id: post-uninstall run: make -j1 -f Makefile purge || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} - name: Post-Clean id: post-z-end run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} TOX: @@ -450,18 +379,21 @@ jobs: timeout-minutes: 30 env: - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.12' LANG: 'en_US.utf-8' steps: - uses: actions/checkout@v4 + with: + submodules: true + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies for Tox run: | - pip install --upgrade "pip>=21.0" "setuptools>=45.0" "wheel>=0.37" "tox>=3.0"; + pip install --upgrade "pip>=24.3.1" "setuptools>=45.0" "wheel>=0.37" "tox>=3.0"; pip install -r ./requirements.txt ; pip install -r ./test-requirements.txt || true ; - name: Pre-Clean @@ -472,4 +404,4 @@ jobs: - name: Post-Clean id: post run: make -j1 -f Makefile clean || true ; - if: ${{ always() }} + if: ${{ !cancelled() }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5439c0f..d243379 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,7 +41,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.2 + with: + submodules: true + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 99a4e23..03061c2 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -34,6 +34,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: + submodules: true persist-credentials: false - name: "Run analysis" diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4b7c06b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "fetch-test-reporter"] + path = includes/fetch-test-reporter + url = https://gist.github.com/c718380a1747d43dda1519167fc50ac0.git + ignore = dirty diff --git a/.hound.yml b/.hound.yml deleted file mode 100644 index 4f2d5e6..0000000 --- a/.hound.yml +++ /dev/null @@ -1,5 +0,0 @@ -python: - enabled: true - config_file: .flake8.ini - -fail_on_violations: true diff --git a/Makefile b/Makefile index cc86c25..588c7bf 100644 --- a/Makefile +++ b/Makefile @@ -17,16 +17,128 @@ # limitations under the License. +ifeq "$(LC_CTYPE)" "" + LC_CTYPE="en_US.UTF-8" +endif + +ifndef SHELL + SHELL:=command -pv bash +endif + +ifeq "$(ERROR_LOG_PATH)" "" + ERROR_LOG_PATH="/dev/null" +endif + +ifeq "$(COMMAND)" "" + COMMAND_CMD=$(shell command -v xcrun || command -v command || command which which || command -v which) + COMMAND_TOOL=$(notdir $(COMMAND_CMD)) + ifeq "$(COMMAND_TOOL)" "xcrun" + COMMAND_ARGS= --find + endif + ifeq "$(COMMAND_TOOL)" "command" + COMMAND_ARGS= -v + endif + ifeq "$(COMMAND_CMD)" "" + COMMAND_CMD="command" + endif + ifdef COMMAND_ARGS + COMMAND := $(COMMAND_CMD) $(COMMAND_ARGS) + else + COMMAND := $(COMMAND_CMD) + endif +endif + +# Check if POSIX touch command can be resolved by COMMAND in the runtime env +ifeq "$(notdir $(shell $(COMMAND) touch))" "" + # This is a non-POSIX environment, so try GHA-Windows-Latest Specific fallback logic + COMMAND="which" +endif + +ifeq "$(MAKE)" "" + # just no cmake please + MAKEFLAGS=$(MAKEFLAGS) -s + MAKE!=`$(COMMAND) make 2>$(ERROR_LOG_PATH) || $(COMMAND) gnumake 2>$(ERROR_LOG_PATH)` +endif + ifeq "$(ECHO)" "" - ECHO=echo + ECHO=printf "%s\n" +endif + +ifdef ACTION + SET_FILE_ATTR=$(shell $(COMMAND) xattr) +endif + +ifdef SET_FILE_ATTR + CREATEDBYBUILDSYSTEM=-w com.apple.xcode.CreatedByBuildSystem true + BSMARK=$(SET_FILE_ATTR) $(CREATEDBYBUILDSYSTEM) +else + BSMARK_CMD=$(shell $(COMMAND) touch) + BSMARK=$(BSMARK_CMD) -a endif ifeq "$(LINK)" "" - LINK=ln -sf + LINK_CMD=$(shell $(COMMAND) ln) + ifneq "$(LINK_CMD)" "" + LINK=$(LINK_CMD) -sf + else + LINK=$(ECHO) "::debug:: Linking is not supported for file: " + endif endif -ifeq "$(MAKE)" "" - MAKE=make +# Python command configuration +ifeq "$(PYTHON)" "" + ifneq "$(PYTHON_VERSION)" "" + PY_CMD=$(shell $(COMMAND) python$(PYTHON_VERSION)) + else + # Try to find python3, fallback to python + PY_CMD=$(shell $(COMMAND) python3) + endif + ifneq "$(PY_CMD)" "" + # Only use -B arg with python3 + PY_ARGS=-B + else + PY_CMD=$(shell $(COMMAND) python) + endif + # Set PYTHON only if not already set + ifdef PY_ARGS + PYTHON := $(PY_CMD) $(PY_ARGS) + else + PYTHON := $(PY_CMD) + endif +endif + +# Coverage configuration +ifeq "$(COVERAGE)" "" + # If PYTHON is set (either by us or externally), try module approach first + ifneq "$(PYTHON)" "" + COVERAGE=$(PYTHON) -m coverage + endif + # If COVERAGE is still not set, fall back to direct command + ifeq "$(COVERAGE)" "" + COVERAGE=$(shell $(COMMAND) coverage) + endif + # Only set COV_CORE_* variables when COVERAGE is configured + ifneq "$(COVERAGE)" "" + #COV_CORE_SOURCE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/ + COV_CORE_CONFIG = $(dir $(abspath $(firstword $(MAKEFILE_LIST)))).coveragerc + COV_CORE_DATAFILE = .coverage + endif +endif + +ifndef PIP_COMMON_FLAGS + # Define common pip install flags + PIP_COMMON_FLAGS := --use-pep517 --exists-action s --upgrade --upgrade-strategy eager +endif + +# Define environment-specific flags +ifeq ($(shell uname -s), Darwin) + PIP_ENV_FLAGS := --break-system-packages + FETCH_CC_INCLUDE_PATH := includes/fetch-test-reporter/tools-fetch-test-reporter.make +else ifeq ($(shell uname -s), Linux) + FETCH_CC_INCLUDE_PATH := includes/fetch-test-reporter/tools-fetch-test-reporter.make +else + FETCH_CC_INCLUDE_PATH := includes\fetch-test-reporter\tools-fetch-test-reporter.make + PIP_ENV_FLAGS := endif ifeq "$(WAIT)" "" @@ -34,7 +146,7 @@ ifeq "$(WAIT)" "" endif ifeq "$(INSTALL)" "" - INSTALL=install + INSTALL=$(shell $(COMMAND) install) ifeq "$(INST_OWN)" "" INST_OWN=-o root -g staff endif @@ -49,96 +161,265 @@ endif ifeq "$(LOG)" "no" QUIET=@ + ifeq "$(DO_FAIL)" "" + DO_FAIL=$(ECHO) "ok" + endif endif ifeq "$(DO_FAIL)" "" - DO_FAIL=$(ECHO) "ok" + DO_FAIL=$(shell $(COMMAND) : ) +endif + +ifeq "$(RM)" "" + RM_CMD=$(shell $(COMMAND) rm) + RM=$(RM_CMD) -f +endif + +ifeq "$(RMDIR)" "" + RMDIR=$(RM)Rd endif -PHONY: must_be_root cleanup +# Include the makefile +include $(FETCH_CC_INCLUDE_PATH) -build: - $(QUIET)$(ECHO) "No need to build. Try make -f Makefile install" - $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile html 2>/dev/null || true +.PHONY: all clean test cleanup init help clean-docs must_be_root must_have_flake must_have_pytest uninstall cleanup-dev-backups + +.env: + $(QUIET)$(ECHO) "Missing environment override." + $(QUIET)$(ECHO) " SHELL: '$(SHELL)'" + $(QUIET)$(ECHO) " LOG: '$(LOG)'" + $(QUIET)$(ECHO) " ERROR_LOG_PATH: '$(ERROR_LOG_PATH)'" + $(QUIET)$(ECHO) " QUIET: '$(QUIET)'" + $(QUIET)$(ECHO) " COMMAND: '$(COMMAND)'" + $(QUIET)$(ECHO) " MAKE: '$(MAKE)'" + $(QUIET)$(ECHO) " ECHO: '$(ECHO)'" + $(QUIET)$(ECHO) " PYTHON: '$(PYTHON)'" + $(QUIET)$(ECHO) " COVERAGE: '$(COVERAGE)'" + $(QUIET)$(ECHO) " BSMARK: '$(BSMARK)'" + $(QUIET)$(ECHO) " FETCH_CC_INCLUDE_PATH: '$(FETCH_CC_INCLUDE_PATH)'" + $(QUIET)$(ECHO) " FETCH_CC_TOOL: '$(FETCH_CC_TOOL)'" + $(QUIET)$(ECHO) " CC_TOOL: '$(CC_TOOL)'" + $(QUIET)$(ECHO) " DS_TOOL: '$(DS_TOOL)'" + $(QUIET)$(ECHO) " CC_TOOL_ARGS: '$(CC_TOOL_ARGS)'" + $(QUIET)$(ECHO) " DS_TOOL_ARGS: '$(DS_TOOL_ARGS)'" + $(QUIET)$(ECHO) " PIP_ENV_FLAGS: '$(PIP_ENV_FLAGS)'" + $(QUIET)$(ECHO) " PIP_COMMON_FLAGS: '$(PIP_COMMON_FLAGS)'" + $(QUIET)$(ECHO) " DO_FAIL: '$(DO_FAIL)'" + $(QUIET)$(ECHO) "" # intentionally blank + +MANIFEST.in: init + $(QUIET)$(ECHO) "include requirements.txt" >"$@" ; + $(QUIET)$(BSMARK) "$@" 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)$(ECHO) "include README.md" >>"$@" ; + $(QUIET)$(ECHO) "include LICENSE.md" >>"$@" ; + $(QUIET)$(ECHO) "include CHANGES.md" >>"$@" ; + $(QUIET)$(ECHO) "include HISTORY.md" >>"$@" ; + $(QUIET)$(ECHO) "recursive-include . *.txt" >>"$@" ; + $(QUIET)$(ECHO) "exclude .gitignore" >>"$@" ; + $(QUIET)$(ECHO) "exclude .deepsource.toml" >>"$@" ; + $(QUIET)$(ECHO) "exclude .*.ini" >>"$@" ; + $(QUIET)$(ECHO) "exclude .*.yml" >>"$@" ; + $(QUIET)$(ECHO) "exclude .*.yaml" >>"$@" ; + $(QUIET)$(ECHO) "global-exclude .git" >>"$@" ; + $(QUIET)$(ECHO) "global-exclude codecov_env" >>"$@" ; + $(QUIET)$(ECHO) "global-exclude .DS_Store" >>"$@" ; + $(QUIET)$(ECHO) "prune .gitattributes" >>"$@" ; + $(QUIET)$(ECHO) "prune test-reports" >>"$@" ; + $(QUIET)$(ECHO) "prune .github" >>"$@" ; + $(QUIET)$(ECHO) "prune .circleci" >>"$@" ; + +build: init ./setup.py MANIFEST.in + $(QUIET)$(PYTHON) -W ignore -m build --sdist --wheel --no-isolation ./ || $(QUIET)$(PYTHON) -W ignore -m build ./ ; + $(QUIET)$(WAIT) + $(QUIET)$(ECHO) "build DONE." init: + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) "pip>=24.3.1" "setuptools>=75.0" "wheel>=0.44" "build>=1.1.1" 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r requirements.txt 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." -install: must_be_root - $(QUIET)python3 -m pip install "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" - $(QUITE)$(WAIT) +install: init build must_be_root + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -e "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "$@: Done." -user-install: - $(QUIET)python3 -m pip install --user "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" - $(QUITE)$(WAIT) +user-install: build + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user "pip>=24.3.1" "setuptools>=75.0" "wheel>=0.44" "build>=1.1.1" 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user -r "https://raw.githubusercontent.com/reactive-firewall/python-repo/stable/requirements.txt" 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) --user -e "git+https://github.com/reactive-firewall/python-repo.git#egg=pythonrepo" + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "$@: Done." uninstall: - $(QUITE)python3 -m pip uninstall pythonrepo || true - $(QUITE)$(WAIT) + $(QUIET)$(PYTHON) -m pip uninstall --use-pep517 $(PIP_ENV_FLAGS) --no-input -y pythonrepo 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(WAIT) $(QUIET)$(ECHO) "$@: Done." -test-reports: - $(QUIET)mkdir test-reports 2>/dev/null >/dev/null || true ; +legacy-purge: clean uninstall + $(QUIET)$(PYTHON) -W ignore ./setup.py uninstall 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(PYTHON) -W ignore ./setup.py clean 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./build/ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./dist/ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./.eggs/ 2>$(ERROR_LOG_PATH) || : + +purge: legacy-purge + $(QUIET)$(RM) ./cc-test-reporter 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./ds-cli.sh 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./bin/deepsource 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./bin/ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./test-reports/*.xml 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || : $(QUIET)$(ECHO) "$@: Done." -purge: clean uninstall - $(QUIET)python3 -m pip uninstall pythonrepo && python -m pip uninstall pythonrepo || true +test-reports: .env + $(QUIET)mkdir $(INST_OPTS) $(dir $(abspath $(firstword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; + $(QUIET)$(BSMARK) $(dir $(abspath $(firstword $(MAKEFILE_LIST))))test-reports 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || true ; $(QUIET)$(ECHO) "$@: Done." -test: cleanup - $(QUIET)coverage run -p --source=pythonrepo -m unittest discover --verbose -s ./tests -t ./ || python3 -m unittest discover --verbose -s ./tests -t ./ || python -m unittest discover --verbose -s ./tests -t ./ || DO_FAIL=exit 2 ; - $(QUIET)coverage combine 2>/dev/null || true - $(QUIET)coverage report -m --include=pythonrepo* 2>/dev/null || true - $(QUIET)$(DO_FAIL); +test-reqs: .env init cc-test-reporter test-reports + $(QUIET)$(PYTHON) -m pip install $(PIP_COMMON_FLAGS) $(PIP_ENV_FLAGS) -r test-requirements.txt 2>$(ERROR_LOG_PATH) || true + +just-test: cleanup + $(QUIET)$(COVERAGE) run -p --source=pythonrepo -m unittest discover --verbose --buffer -s $(dir $(abspath $(firstword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) || $(PYTHON) -m unittest discover --verbose --buffer -s $(dir $(abspath $(firstword $(MAKEFILE_LIST))))tests -t $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; + $(QUIET)$(WAIT) ; + $(QUIET)$(DO_FAIL) ; + +test: test-reqs just-test + $(QUIET)$(DO_FAIL) ; + $(QUIET)$(COVERAGE) combine 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(COVERAGE) report -m --include=pythonrepo/* 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; $(QUIET)$(ECHO) "$@: Done." test-tox: cleanup $(QUIET)tox -v -- || tail -n 500 .tox/py*/log/py*.log 2>/dev/null $(QUIET)$(ECHO) "$@: Done." -test-pytest: cleanup test-reports - $(QUIET)python3 -m pytest --junitxml=test-reports/junit.xml -v tests || python -m pytest --junitxml=test-reports/junit.xml -v tests +test-pytest: cleanup MANIFEST.in test-reqs must_have_pytest test-reports + $(QUIET)$(PYTHON) -m pytest --cache-clear --doctest-glob=pythonrepo/*.py --doctest-modules --cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml -v --rootdir=$(dir $(abspath $(firstword $(MAKEFILE_LIST)))) || DO_FAIL="exit 2" ; + $(QUIET)$(CC_TOOL) $(CC_TOOL_ARGS) 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(CA_TOOL) $(CA_TOOL_ARGS) || : ; + $(QUIET)$(DS_TOOL) $(DS_TOOL_ARGS) || : ; + $(QUIET)$(WAIT) ; + $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." test-style: cleanup - $(QUIET)python3 -m flake8 --ignore=W191,W391 --max-line-length=100 --verbose --count --config=.flake8.ini --show-source || DO_FAIL="exit 2" ; + $(QUIET)$(PYTHON) -m flake8 --ignore=W191,W391 --max-line-length=100 --verbose --count --config=.flake8.ini --show-source || DO_FAIL="exit 2" ; + $(QUIET)tests/check_cc_lines 2>/dev/null || true $(QUIET)tests/check_spelling 2>/dev/null || true $(QUIET)$(ECHO) "$@: Done." -cleanup: - $(QUIET)rm -f tests/*.pyc 2>/dev/null || true - $(QUIET)rm -f tests/*~ 2>/dev/null || true - $(QUIET)rm -Rf tests/__pycache__ 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*.pyc 2>/dev/null || true - $(QUIET)rm -Rf pythonrepo/__pycache__ 2>/dev/null || true - $(QUIET)rm -Rf pythonrepo/*/__pycache__ 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*~ 2>/dev/null || true - $(QUIET)rm -f *.pyc 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*/*.pyc 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*/*~ 2>/dev/null || true - $(QUIET)rm -f *.DS_Store 2>/dev/null || true - $(QUIET)rm -Rf .pytest_cache/ 2>/dev/null || true - $(QUIET)rmdir ./test-reports/ 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*.DS_Store 2>/dev/null || true - $(QUIET)rm -f pythonrepo/*/*.DS_Store 2>/dev/null || true - $(QUIET)rm -f pythonrepo.egg-info/* 2>/dev/null || true - $(QUIET)rmdir pythonrepo.egg-info 2>/dev/null || true - $(QUIET)rm -f ./*/*~ 2>/dev/null || true - $(QUIET)rm -f ./*~ 2>/dev/null || true - $(QUIET)coverage erase 2>/dev/null || true - $(QUIET)rm -f ./.coverage 2>/dev/null || true - $(QUIET)rm -f ./coverage*.xml 2>/dev/null || true - $(QUIET)rm -f ./sitecustomize.py 2>/dev/null || true - $(QUIET)rm -f ./.*~ 2>/dev/null || true - $(QUIET)rm -Rf ./.tox/ 2>/dev/null || true - -clean: cleanup - $(QUIET)rm -f test-results/junit.xml 2>/dev/null || true - $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile clean || true +cc-test-reporter: $(FETCH_CC_INCLUDE_PATH) + $(QUIET)$(FETCH_CC_TOOL) || DO_FAIL="exit 2" ; + $(QUIET)$(WAIT) ; + $(QUIET)$(DO_FAIL) ; $(QUIET)$(ECHO) "$@: Done." +must_have_flake: + $(QUIET)runner=`$(PYTHON) -m pip freeze --all | grep --count -oF flake` ; \ + if test $$runner -le 0 ; then $(ECHO) "No Linter found for test." ; exit 126 ; fi + +must_have_pytest: init + $(QUIET)runner=`$(PYTHON) -m pip freeze --all | grep --count -oF pytest` ; \ + if test $$runner -le 0 ; then $(ECHO) "No python framework (pytest) found for test." ; exit 126 ; fi + +cleanup-cc-test-reporter: $(FETCH_CC_INCLUDE_PATH) + $(QUIET)$(CLEAN_CC_TOOL) || : + +cleanup-dev-backups:: + $(QUIET)$(RM) ./*/*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./.*/*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./**/*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*~ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./.*~ 2>$(ERROR_LOG_PATH) || : + +cleanup-mac-dir-store:: + $(QUIET)$(RM) ./.DS_Store 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/.DS_Store 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/.DS_Store 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/**/.DS_Store 2>$(ERROR_LOG_PATH) || : + +cleanup-py-caches: cleanup-dev-backups cleanup-mac-dir-store + $(QUIET)$(RM) ./*.pyc 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/*.pyc 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/__pycache__/* 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RM) ./*/*/*.pyc 2>$(ERROR_LOG_PATH) || : + +cleanup-py-cache-dirs: cleanup-py-caches + $(QUIET)$(RMDIR) ./tests/__pycache__ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./*/__pycache__ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./*/*/__pycache__ 2>$(ERROR_LOG_PATH) || : + $(QUIET)$(RMDIR) ./__pycache__ 2>$(ERROR_LOG_PATH) || : + +cleanup-hypothesis:: + $(QUIET)$(RM) ./.hypothesis/**/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./.hypothesis/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./.hypothesis/ 2>$(ERROR_LOG_PATH) || true + +cleanup-tests: cleanup-hypothesis cleanup-py-cache-dirs cleanup-py-caches + $(QUIET)$(RM) ./test_env/**/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./test_env/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./test_env/ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) .pytest_cache/ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./.tox/ 2>$(ERROR_LOG_PATH) || true + +cleanup-pythonrepo: cleanup-py-cache-dirs cleanup-py-caches + $(QUIET)$(RM) pythonrepo/*.pyc 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) pythonrepo/*~ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) pythonrepo/__pycache__/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) pythonrepo/*/*.pyc 2>$(ERROR_LOG_PATH) || true + +cleanup-pythonrepo-eggs: cleanup-dev-backups cleanup-mac-dir-store + $(QUIET)$(RM) ./*.egg-info/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./*.egg-info 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) .eggs 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./.eggs/ 2>$(ERROR_LOG_PATH) || true + +cleanup-src-dir: cleanup-dev-backups cleanup-mac-dir-store + $(QUIET)$(RM) ./src/**/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./src/* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./src/ 2>$(ERROR_LOG_PATH) || true + +cleanup: cleanup-tests cleanup-pythonrepo cleanup-pythonrepo-eggs cleanup-src-dir + $(QUIET)$(RM) ./.coverage 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./coverage*.xml 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./.coverage.* 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./sitecustomize.py 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RMDIR) ./test-reports/ 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(WAIT) ; + +build-docs: ./docs/ ./docs/Makefile docs-reqs + $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile html 2>$(ERROR_LOG_PATH) || DO_FAIL="exit 2" ; + $(QUIET)$(WAIT) ; + $(QUIET)mkdir $(INST_OPTS) ./docs/www 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || : ; + $(QUIET)$(BSMARK) ./docs/www 2>$(ERROR_LOG_PATH) >$(ERROR_LOG_PATH) || : ; + $(QUIET)$(WAIT) ; + $(QUIET)cp -fRp ./docs/_build/ ./docs/www/ 2>$(ERROR_LOG_PATH) || DO_FAIL="exit 35" ; + $(QUIET)$(WAIT) ; + $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile clean 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(WAIT) ; + $(QUIET)$(ECHO) "Documentation should be in docs/www/html/" + $(QUIET)$(DO_FAIL) ; + +clean-docs: ./docs/ ./docs/Makefile + $(QUIET)$(RM) ./docs/www/* 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(RMDIR) ./docs/www/ 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(MAKE) -s -C ./docs/ -f Makefile clean 2>$(ERROR_LOG_PATH) || : ; + $(QUIET)$(WAIT) ; + +./docs/: + $(QUIET) : ; + +./docs/Makefile: ./docs/ + $(QUIET)$(WAIT) ; + +clean: clean-docs cleanup cleanup-cc-test-reporter + $(QUIET)$(COVERAGE) erase 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./test-results/junit.xml 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(RM) ./MANIFEST.in 2>$(ERROR_LOG_PATH) || true + $(QUIET)$(ECHO) "All clean." + must_be_root: $(QUIET)runner=`whoami` ; \ if test $$runner != "root" ; then echo "You are not root." ; exit 1 ; fi diff --git a/docs/conf.py b/docs/conf.py index 23e6b33..314411b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,7 @@ '.rst': 'restructuredtext', } -# The encoding of source files. Official sphinx docs reccomend utf-8-sig. +# The encoding of source files. Official sphinx docs recommend utf-8-sig. source_encoding = 'utf-8-sig' # The master toctree document. diff --git a/includes/fetch-test-reporter b/includes/fetch-test-reporter new file mode 160000 index 0000000..d72e1f8 --- /dev/null +++ b/includes/fetch-test-reporter @@ -0,0 +1 @@ +Subproject commit d72e1f89868d8d6a170c847ae3b3b19866042ab4 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e95f439 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["setuptools>=75.0", "build>=1.2.1", "wheel>=0.44"] +build-backend = "setuptools.build_meta" + +[pytest.enabler.flake8] +addopts = "--flake8" + +[pytest.enabler.doctest] +addopts = "--doctest-glob=**/*.py --doctest-modules" + +[pytest.enabler.cov] +addopts = "--cov=. --cov-append --cov-report=xml --junitxml=test-reports/junit.xml" diff --git a/pythonrepo/__init__.py b/pythonrepo/__init__.py index b24afa4..40154f0 100644 --- a/pythonrepo/__init__.py +++ b/pythonrepo/__init__.py @@ -30,7 +30,7 @@ try: import sys import os - if str(__module__) in __file__: + if str(__module__) in __file__: # pragma: no branch __parentPath = os.path.join( os.path.dirname(__file__), '..' ) diff --git a/pythonrepo/pythonrepo.py b/pythonrepo/pythonrepo.py index e20e95a..e785263 100644 --- a/pythonrepo/pythonrepo.py +++ b/pythonrepo/pythonrepo.py @@ -102,7 +102,7 @@ def useTool(tool, *arguments): try: TASK_OPTIONS[tool](arguments) except Exception: - w = str("WARNING - An error occured while") + w = str("WARNING - An error occurred while") w += str("handling the tool. Abort.") print(w) else: @@ -117,13 +117,13 @@ def main(*argv): service_cmd = args.some_task useTool(service_cmd, extra) except Exception: - w = str("WARNING - An error occured while") + w = str("WARNING - An error occurred while") w += str("handling the arguments.") w += str(" Cascading failure.") print(w) sys.exit(2) except Exception: - e = str("CRITICAL - An error occured while handling") + e = str("CRITICAL - An error occurred while handling") e += str("the cascading failure.") print(e) sys.exit(3) diff --git a/requirements.txt b/requirements.txt index 6841f4b..2757206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,18 +4,21 @@ # subprocess - PSF licence # sphinx # argparse - builtin - PSF licence -argparse>=1.4.0 -# argparse - builtin - PSF licence -setuptools>=38.0 -# virtualenv - MIT -virtualenv>=15.0.1 # six - MIT six>=1.0.0 # pgpy - BSD 3-Clause licensed #pgpy>=0.4.1 tox>=3.0.0 #py>=1.4.33 -pip>=19.0 +# setuptools - MIT license +setuptools>=75.0 +# virtualenv - MIT license +virtualenv>=20.26.6 +# pip - MIT license +pip>=24.3.1 +# build - MIT license +build>=1.1.1, !=1.2.2.post1 # python-repo - MIT #-e git+https://github.com/reactive-firewall/python-repo.git#egg=python-repo - +# wheel - MIT license +wheel>=0.44 diff --git a/setup.cfg b/setup.cfg index 7487dc8..209f6d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,50 +12,41 @@ classifiers = Development Status :: 4 - Beta Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux - Programming Language :: Python + License :: OSI Approved :: MIT License Programming Language :: Python :: 3 - Programming Language :: Python :: 3.14 - Programming Language :: Python :: 3.13 Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.4 - Programming Language :: Python :: 3.3 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 :: Only Topic :: Software Development :: Libraries :: Python Modules - Topic :: Security + Topic :: System :: Networking license = MIT license_files = - LICENSE[.md]* + LICENSE.md platform = any project_urls = - Bug Tracker = https://github.com/reactive-firewall/python-repo/issues + "Bug Tracker" = https://github.com/reactive-firewall/python-repo/issues License = https://github.com/reactive-firewall/python-repo/LICENSE.md + Documentation = https://reactive-firewallpython-repo.readthedocs.io/en/stable + Repository = https://github.com/reactive-firewall/python-repo.git [bdist_rpm] url = https://github.com/reactive-firewall/python-repo.git [bdist_wheel] -universal=1 - -[files] -packages = pythonrepo +universal=0 [options] -zip_safe = true +zip_safe = false py_modules = pythonrepo test_suite = tests -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.15.* +python_requires = >=3.9.20 setup_requires = - setuptools>=45.0.0 - wheel>=0.37.0 - build>=1.1.1 + setuptools>=75.0.0 + wheel>=0.44.0 + build>=1.2.1 packages = find: @@ -66,7 +57,7 @@ where = tests/ *.py include = - pythonrepo + pythonrepo.py exclude = docs tests @@ -74,16 +65,19 @@ exclude = [options.extras_require] testing = # upstream - pytest >= 7 + pytest >= 7.4 pytest-checkdocs >= 2.4 - pytest-flake8 - coverage >= 6.3 - pytest-cov >= 4.0.0; \ - # coverage seems to make PyPy extremely slow - python_implementation != "PyPy" + pytest-flake8 >= 1.0.7 + coverage >= 7 pytest-enabler >= 1.0.1 # local flake8 >= 5.0 - virtualenv >= 15.0.1 - wheel >= 0.37.0 - pip >= 21.0 + virtualenv >= 20.26.6 + wheel >= 0.44.0 + pip >= 24.3.1 + pytest-cov >= 4.0.0; \ + # coverage seems to make PyPy extremely slow + python_implementation != "PyPy" + +[tool:pytest] +addopts = --doctest-glob=**/*.py --doctest-modules --cov= --cov-append --cov-report=xml diff --git a/setup.py b/setup.py index e852299..ce76a51 100755 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ def readFile(filename): str("""Topic :: Software Development :: Libraries :: Python Modules""") ] except Exception: - class_tags = str("""Development Status :: 4 - Beta""") + class_tags = [str("""Development Status :: 4 - Beta""")] setup( name=conf_dict["""metadata"""]["""name"""], diff --git a/test-requirements.txt b/test-requirements.txt index b09086e..6ede96c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,19 +3,24 @@ # re - builtin - PSF licence? # subprocess - PSF licence # sphinx -# argparse - builtin - PSF licence -argparse>=1.4.0 -# argparse - builtin - PSF licence -setuptools>=38.0 -# virtualenv - MIT -virtualenv>=15.0.1 # six - MIT six>=1.0.0 # pgpy - BSD 3-Clause licensed #pgpy>=0.4.1 tox>=3.0.0 #py>=1.4.33 -# pip>=19.0 +# setuptools - MIT license +setuptools>=75.0 +# virtualenv - MIT license +virtualenv>=20.26.6 +# pip - MIT license +pip>=24.3.1 +# build - MIT license +build>=1.1.1, !=1.2.2.post1 +# python-repo - MIT +#-e git+https://github.com/reactive-firewall/python-repo.git#egg=python-repo +# wheel - MIT license +wheel>=0.44 # # TESTING ONLY - Do NOT report issues with these optionals on python-repo # @@ -30,5 +35,3 @@ pytest-flake8>=1.0 coverage >= 6.3 pytest-cov >= 4.0.0; pytest-enabler >= 1.0.1 -wheel >= 0.37.0 -pip >= 21.0 diff --git a/tests/fetch_cc-test-reporter b/tests/check_cc_lines similarity index 59% rename from tests/fetch_cc-test-reporter rename to tests/check_cc_lines index c84b235..722272b 100755 --- a/tests/fetch_cc-test-reporter +++ b/tests/check_cc_lines @@ -60,37 +60,49 @@ # even if the above stated remedy fails of its essential purpose. ################################################################################ -ulimit -t 1200 -PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${PATH}" -LANG=${LANG:-"en_US"} -LC_ALL="${LANG:1:5}.utf-8" -umask 127 - -LOCK_FILE="${TMPDIR:-/tmp}/org.pak.tests.scripts.code-climate.lock" -EXIT_CODE=0 - -test -x "$(command -v grep)" || exit 126 ; -test -x "$(command -v curl)" || exit 126 ; -hash -p ./.github/tool_shlock_helper.sh shlock || exit 255 ; -test -x "$(command -v shlock)" || exit 126 ; -test -x "$(command -v gpgv)" || exit 126 ; -test -x "$(command -v shasum)" || exit 126 ; - -# sorry no windows support here -if [[ $( command uname -s ) == *arwin ]] ; then - CI_OS="darwin" -else - CI_OS="linux" -fi +ulimit -t 600 +PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" +umask 137 + +LOCK_FILE="${TMPDIR:-/tmp}/cc_line_test_script_lock" +EXIT_CODE=1 + +# Function to check if a command exists. + +# USAGE: +# ~$ check_command CMD +# Arguments: +# CMD (Required) -- Name of the command to check +# Results: +# exits 64 -- missing required argument +# exits 126 -- check complete and has failed, can not find given command. +# returns successful -- check complete and command found to be executable. +function check_command() { + test -z "$1" && { printf "%s\n" "Error: command name is required to check for existence." >&2 ; exit 64 ; } ; + local cmd="$1" ; + test -x "$(command -v ${cmd})" || { printf "%s\n" "Error: Required command '$cmd' is not found." >&2 ; exit 126 ; } ; +} # end check_command() +# propagate/export function to sub-shells +export -f check_command + +# Set up CEP-5 shlock helper +hash -p ./.github/tool_shlock_helper.sh shlock || { printf "%s\n" "Error: Failed to register shlock helper. CEP-5 locking will not work." >&2 ; exit 78 ; } ; + +# Check required commands +check_command grep ; +check_command curl ; +check_command find ; +check_command git ; +check_command shlock ; + function cleanup() { - rm -f "${LOCK_FILE}" 2>/dev/null || true ; wait ; - rm -f ./test-reporter-latest-*-amd64.SHA*.sig 2>/dev/null || true ; wait ; - rm -f ./test-reporter-latest-*-amd64.SHA* 2>/dev/null > /dev/null || true ; wait ; + rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; hash -d shlock 2>/dev/null > /dev/null || true ; } -if [[ ( $(shlock -f "${LOCK_FILE}" -p $$ ) -eq 0 ) ]] ; then +if [[ ( $(shlock -f ${LOCK_FILE} -p $$ ) -eq 0 ) ]] ; then + EXIT_CODE=0 trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 @@ -99,73 +111,56 @@ if [[ ( $(shlock -f "${LOCK_FILE}" -p $$ ) -eq 0 ) ]] ; then trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true || true ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 - trap 'cleanup ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 else - # shellcheck disable=SC2046 - printf "\t%s\n" "CodeClimate already in progress by "$(head "${LOCK_FILE}") >&2 ; + printf "%s\n" "Check for Copyright lines already in progress by "`head ${LOCK_FILE}` ; false ; exit 126 ; fi # this is how test files are found: +_TEST_FILE_VALIDATOR="echo " -# THIS IS THE ACTUAL TEST DIR USED (update _TEST_ROOT_DIR as needed) -_TEST_ROOT_DIR="./" ; -if [[ -d ./.git ]] ; then - _TEST_ROOT_DIR="./" ; -elif [[ -d ./tests ]] ; then - _TEST_ROOT_DIR="./" ; +# THIS IS THE ACTUAL TEST +if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); then + if [ -z "${_TEST_ROOT_DIR}" ]; then + _TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) + fi else - printf "%s\n" "FAIL: missing valid folder or file" - EXIT_CODE=1 -fi - -# This File MUST BE GIT-IGNORED -# to be SAFELY USED to store Tokens and env vars (update logic as needed) -if [[ ( -r ./codecov_env ) ]] ; then - # shellcheck disable=SC2086,SC1091 - source ./codecov_env 2>/dev/null || true ; + printf "\t%s\n" "FAIL: missing valid repository or source structure" >&2 + EXIT_CODE=40 fi +_TEST_YEAR=$(date "+%Y" 2>/dev/null || date -j "+%C%y" 2>/dev/null ;) -######################### -# actual Work starts here -######################### -curl -fLso ./test-reporter-latest-${CI_OS:-linux}-amd64 https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-amd64 ; -for i in 1 256 512 ; do - curl -fLso test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}" ; wait ; - curl -fLso test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig" ; wait ; - # test sha1/sha512 signatures if found and sha256 even if not found - if [[ ( -r test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} ) ]] || [[ ( ${i} -eq 256 ) ]] ; then - if [[ ${i} -eq 1 ]]; then - printf "%s\n" "WARNING: SHA-1 is deprecated and should be avoided when possible. Consider using SHA-256 or SHA-512 for stronger integrity checks. (CWE-327: Use of a Broken or Risky Cryptographic Algorithm)" +for _TEST_DOC in $(find ${_TEST_ROOT_DIR} \( -not -ipath "*.github/*" \) -a \( -iname '*.py' -o -iname '*.txt' -o -iname '*.md' \) -a -print0 2>&1 | xargs -0 -I{} ${_TEST_FILE_VALIDATOR} "{}" ; wait ;) ; do + if [[ ($(grep -cF 'Disclaimer' "${_TEST_DOC}" 2>&1 || : ;) -ne 0) ]] ; then + printf "%s\n" "SKIP: ${_TEST_DOC} is disclaimed." ; + EXIT_CODE=126; + else + if [[ ($(grep -cF "Copyright" "${_TEST_DOC}" 2>&1 || : ;) -le 0) ]] ; then + printf "%s\n" "FAIL: ${_TEST_DOC} is missing a copyright line" >&2 ; + EXIT_CODE=127 + fi + if [[ ( $(grep -F "Copyright" "${_TEST_DOC}" 2>&1 | grep -coF "Copyright (c)" 2>&1) -le 0) ]] ; then + printf "%s\n" "SKIP: ${_TEST_DOC} is missing a valid copyright line begining with \"Copyright (c)\"" ; fi - if [[ ( -r test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig ) ]] ; then - # configure your CI evironment to trust the key at gpg --keyserver keys.openpgp.org --recv-keys 9BD9E2DD46DA965A537E5B0A5CBF320243B6FD85 - # FP: KEY FP 9BD9 E2DD 46DA 965A 537E 5B0A 5CBF 3202 43B6 FD85 - # OR... - # Set CI=true to continue on missing keys - gpg --verify test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} || ${CI} || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_OS:-linux}-amd64.sha${i}.sig 2>/dev/null ; + if [[ ( $(grep -F "Copyright (c)" "${_TEST_DOC}" 2>&1 | grep -oE "\d+(-\d+)?" 2>&1 | grep -oE "\d{3,}$" | sort -n | tail -n1) -lt ${_TEST_YEAR}) ]] ; then + printf "%s\n" "WARN: ${_TEST_DOC} is out of date without a current copyright (year)" >&2 ; + fi + if [[ ( ${EXIT_CODE} -ne 0 ) ]] ; then + case "$EXIT_CODE" in + 0|126) true ;; + *) printf "%s\n" "SKIP: Unclassified issue with '${_TEST_DOC}'" ;; + esac fi - shasum -a $i -c --ignore-missing test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} || EXIT_CODE=126 - rm -vf test-reporter-latest-${CI_OS:-linux}-amd64.sha${i} 2>/dev/null ; fi done -if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - mv -f test-reporter-latest-${CI_OS:-linux}-amd64 ./cc-test-reporter 2>/dev/null || EXIT_CODE=126 - chmod -v 751 ./cc-test-reporter || EXIT_CODE=126 -fi - -if [[ ( ${EXIT_CODE} -eq 0 ) ]] ; then - ./cc-test-reporter before-build || EXIT_CODE=10 ; -fi - - unset _TEST_ROOT_DIR 2>/dev/null || true ; +unset _TEST_DOC 2>/dev/null || true ; +unset _TEST_YEAR 2>/dev/null || true ; -cleanup 2>/dev/null || rm -f "${LOCK_FILE}" 2>/dev/null > /dev/null || true ; wait ; +rm -f ${LOCK_FILE} 2>/dev/null > /dev/null || true ; wait ; # goodbye exit ${EXIT_CODE:-255} ; diff --git a/tests/check_spelling b/tests/check_spelling index a6bc29c..15b0e77 100755 --- a/tests/check_spelling +++ b/tests/check_spelling @@ -1,19 +1,214 @@ #! /bin/bash +# Disclaimer of Warranties. +# A. YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT, TO THE EXTENT PERMITTED BY +# APPLICABLE LAW, USE OF THIS SHELL SCRIPT AND ANY SERVICES PERFORMED +# BY OR ACCESSED THROUGH THIS SHELL SCRIPT IS AT YOUR SOLE RISK AND +# THAT THE ENTIRE RISK AS TO SATISFACTORY QUALITY, PERFORMANCE, ACCURACY AND +# EFFORT IS WITH YOU. +# +# B. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THIS SHELL SCRIPT +# AND SERVICES ARE PROVIDED "AS IS" AND "AS AVAILABLE", WITH ALL FAULTS AND +# WITHOUT WARRANTY OF ANY KIND, AND THE AUTHOR OF THIS SHELL SCRIPT'S LICENSORS +# (COLLECTIVELY REFERRED TO AS "THE AUTHOR" FOR THE PURPOSES OF THIS DISCLAIMER) +# HEREBY DISCLAIM ALL WARRANTIES AND CONDITIONS WITH RESPECT TO THIS SHELL SCRIPT +# SOFTWARE AND SERVICES, EITHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES AND/OR CONDITIONS OF +# MERCHANTABILITY, SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE, +# ACCURACY, QUIET ENJOYMENT, AND NON-INFRINGEMENT OF THIRD PARTY RIGHTS. +# +# C. THE AUTHOR DOES NOT WARRANT AGAINST INTERFERENCE WITH YOUR ENJOYMENT OF THE +# THE AUTHOR's SOFTWARE AND SERVICES, THAT THE FUNCTIONS CONTAINED IN, OR +# SERVICES PERFORMED OR PROVIDED BY, THIS SHELL SCRIPT WILL MEET YOUR +# REQUIREMENTS, THAT THE OPERATION OF THIS SHELL SCRIPT OR SERVICES WILL +# BE UNINTERRUPTED OR ERROR-FREE, THAT ANY SERVICES WILL CONTINUE TO BE MADE +# AVAILABLE, THAT THIS SHELL SCRIPT OR SERVICES WILL BE COMPATIBLE OR +# WORK WITH ANY THIRD PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES, +# OR THAT DEFECTS IN THIS SHELL SCRIPT OR SERVICES WILL BE CORRECTED. +# INSTALLATION OF THIS THE AUTHOR SOFTWARE MAY AFFECT THE USABILITY OF THIRD +# PARTY SOFTWARE, APPLICATIONS OR THIRD PARTY SERVICES. +# +# D. YOU FURTHER ACKNOWLEDGE THAT THIS SHELL SCRIPT AND SERVICES ARE NOT +# INTENDED OR SUITABLE FOR USE IN SITUATIONS OR ENVIRONMENTS WHERE THE FAILURE +# OR TIME DELAYS OF, OR ERRORS OR INACCURACIES IN, THE CONTENT, DATA OR +# INFORMATION PROVIDED BY THIS SHELL SCRIPT OR SERVICES COULD LEAD TO +# DEATH, PERSONAL INJURY, OR SEVERE PHYSICAL OR ENVIRONMENTAL DAMAGE, +# INCLUDING WITHOUT LIMITATION THE OPERATION OF NUCLEAR FACILITIES, AIRCRAFT +# NAVIGATION OR COMMUNICATION SYSTEMS, AIR TRAFFIC CONTROL, LIFE SUPPORT OR +# WEAPONS SYSTEMS. +# +# E. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY THE AUTHOR +# SHALL CREATE A WARRANTY. SHOULD THIS SHELL SCRIPT OR SERVICES PROVE DEFECTIVE, +# YOU ASSUME THE ENTIRE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. +# +# Limitation of Liability. +# F. TO THE EXTENT NOT PROHIBITED BY APPLICABLE LAW, IN NO EVENT SHALL THE AUTHOR +# BE LIABLE FOR PERSONAL INJURY, OR ANY INCIDENTAL, SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES WHATSOEVER, INCLUDING, WITHOUT LIMITATION, DAMAGES +# FOR LOSS OF PROFITS, CORRUPTION OR LOSS OF DATA, FAILURE TO TRANSMIT OR +# RECEIVE ANY DATA OR INFORMATION, BUSINESS INTERRUPTION OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES, ARISING OUT OF OR RELATED TO YOUR USE OR +# INABILITY TO USE THIS SHELL SCRIPT OR SERVICES OR ANY THIRD PARTY +# SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH THIS SHELL SCRIPT OR +# SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY OF LIABILITY (CONTRACT, +# TORT OR OTHERWISE) AND EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION +# OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF INCIDENTAL OR +# CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU. In no event +# shall THE AUTHOR's total liability to you for all damages (other than as may +# be required by applicable law in cases involving personal injury) exceed +# the amount of five dollars ($5.00). The foregoing limitations will apply +# even if the above stated remedy fails of its essential purpose. +################################################################################ +# +# This script attempts to enforce spell-checking if the codespell tool is available. +# It accomplishes the following tasks: +# 1. Sets up error handling and cleanup mechanisms to ensure proper execution. +# 2. Determines the test root directory based on the project structure. +# 3. lint and check spelling +# 4. cleanup and report +# +# Usage Summary: +# To lint the project without making changes: +# ./tests/check_spelling +# To lint and auto-correct spelling errors: +# ./tests/check_spelling --fix +# +# Exit Code Summary: +# The script uses the EXIT_CODE variable to track exit conditions: +# - 0: Successful execution. +# - 1: General failure. +# - 2: Coverage combine or XML generation failed. +# - 3: git ls-tree command failed. +# - 40: Missing valid repository or source structure. +# - 126: Script already in progress or command not executable. +# - 129: Received SIGHUP signal. +# - 130: Received SIGINT signal (Ctrl+C). +# - 131: Received SIGQUIT signal. +# - 137: Received SIGABRT signal. +# - 143: Received SIGTERM signal. +# +# The primary goal is to allow linter like spell-checking. -# exit fast if command is missing -test -x /usr/bin/spellintian || exit 0 ; +ulimit -t 600 +# setting the path may break brain-dead CI that uses crazy paths +# PATH="/bin:/sbin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" +umask 137 -THE_TEMP_FILE="/tmp/swapfile_spellcheck_${RANDOM}.tmp.txt" ; -( (spellintian "${@:-./**/*}" 2>/dev/null | fgrep -v "(duplicate word)" | fgrep " -> ") & (spellintian "${@:-./*}" 2>/dev/null | fgrep -v "(duplicate word)" | fgrep " -> ") & (spellintian "${@:-./**/**/*}" 2>/dev/null | fgrep -v "(duplicate word)" | fgrep " -> ") ) | sort -h | uniq | tee -a ${THE_TEMP_FILE:-/dev/null} ; -wait ; -THECOUNT=$( (wc -l ${THE_TEMP_FILE} 2>/dev/null || echo 0) | cut -d\ -f 1 ) ; -EXIT_CODE=${THECOUNT} ; -if [[ ("${THECOUNT}" -le 1) ]] ; then - EXIT_CODE=0 ; - echo "OK: Found no detected spelling errors." ; +# force utf-8 for spelling +export LC_CTYPE="${LC_CTYPE:-en_US.UTF-8}" + +LOCK_FILE="${TMPDIR:-/tmp}/org.pak.multicast.spell-check-shell" +EXIT_CODE=1 + +# Function to check if a command exists. + +# USAGE: +# ~$ check_command CMD +# Arguments: +# CMD (Required) -- Name of the command to check +# Results: +# exits 64 -- missing required argument +# exits 126 -- check complete and has failed, can not find given command. +# returns successful -- check complete and command found to be executable. +function check_command() { + test -z "$1" && { printf "%s\n" "Error: command name is required to check for existence." >&2 ; exit 64 ; } ; + local cmd="$1" ; + test -x "$(command -v ${cmd})" || { printf "%s\n" "Error: Required command '$cmd' is not found." >&2 ; exit 126 ; } ; +} # end check_command() +# propagate/export function to sub-shells +export -f check_command + +# Set up CEP-5 shlock helper +hash -p ./.github/tool_shlock_helper.sh shlock || { printf "%s\n" "Error: Failed to register shlock helper. CEP-5 locking will not work." >&2 ; exit 78 ; } ; + +# Check required commands +check_command grep ; +check_command python3 ; +check_command codespell ; +check_command git ; +check_command shlock ; + + +SCRIPT_FILE="tests/check_spelling" + +# Set codespell options +CODESPELL_OPTIONS="--quiet-level=4 --builtin clear,rare,code -L assertIn" + +function cleanup() { + rm -f ${LOCK_FILE} 2>/dev/null || : ; wait ; + # unset when done + unset LOCK_FILE 2>/dev/null || : ; + hash -d shlock 2>/dev/null || : ; +} + +if [[ ( $(shlock -f ${LOCK_FILE} -p $$ ) -eq 0 ) ]] ; then + EXIT_CODE=0 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 129 ;' SIGHUP || EXIT_CODE=129 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 143 ;' SIGTERM || EXIT_CODE=143 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 131 ;' SIGQUIT || EXIT_CODE=131 + # SC2173 - https://github.com/koalaman/shellcheck/wiki/SC2173 + #trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 1 ;' SIGSTOP || EXIT_CODE=7 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit 130 ;' SIGINT || EXIT_CODE=130 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true || true ; wait ; exit 137 ;' SIGABRT || EXIT_CODE=137 + trap 'cleanup 2>/dev/null || rm -f ${LOCK_FILE} 2>/dev/null || true ; wait ; exit ${EXIT_CODE} ;' EXIT || EXIT_CODE=1 else - echo "FAIL: Found ${THECOUNT:-many} spelling errors." ; + # shellcheck disable=SC2046 + printf "\t%s\n" "Check Setup Scripts Tests Coverage already in progress by "$(head "${LOCK_FILE}") >&2 ; + exit 126 ; fi -rm -f ${THE_TEMP_FILE} 2>/dev/null >> /dev/null || true ; + +function report_summary() { + printf "::group::%s\n" "Results" ; + # Improved reporting based on EXIT_CODE + case "${EXIT_CODE}" in + 0) printf "::notice title=OK::%s\n" "OK: Found no detected spelling errors." ;; + 1) printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=CHECK-SPELLING::%s\n" "FAIL: General failure during script execution." >&2 ;; + 3) printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=CONFIGURATION::%s\n" "FAIL: Gathering repostory's requirements failed." >&2 ;; # git ls-tree command failed + 126) printf "::warning file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=SKIPPED::%s\n" "SKIP: Unable to continue script execution." >&2 ;; + *) printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=FAILED::%s\n" "FAIL: Detected spelling errors." >&2 ;; + esac + printf "::endgroup::\n" ; +} + +# this is how test files are found: + +# THIS IS THE ACTUAL TEST +_TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) ; +if _TEST_ROOT_DIR=$(git rev-parse --show-superproject-working-tree 2>/dev/null); then + if [ -z "${_TEST_ROOT_DIR}" ]; then + _TEST_ROOT_DIR=$(git rev-parse --show-toplevel 2>/dev/null) + fi + printf "::debug::%s\n" "Found ${_TEST_ROOT_DIR} ..." ; + else + printf "::error file=${SCRIPT_FILE},line=${BASH_LINENO:-0},title=${FUNCNAME:-$0}::%s\n" "FAIL: missing valid repository or source structure" >&2 + EXIT_CODE=40 +fi + +# Get a list of files to check using git ls-tree with filtering +FILES_TO_CHECK=$(git ls-tree -r --full-tree --name-only HEAD -- *.md *.py *.txt **/*.md **/*.txt **/*.py 2>/dev/null || EXIT_CODE=3) + +# Enable auto-correction if '--fix' argument is provided +if [[ "$1" == "--fix" ]]; then + CODESPELL_OPTIONS="--write-changes --interactive 2 ${CODESPELL_OPTIONS}" + printf "::debug::%s\n" "Auto-correction enabled." +fi + +# THIS IS THE ACTUAL TEST +# Iterate over files and run codespell +for FILE in $FILES_TO_CHECK; do + printf "::group::%s\n" "Checking ${FILE}" ; + { codespell $CODESPELL_OPTIONS "${FILE}" || EXIT_CODE=$? ;} 2>/dev/null ; wait ; + printf "::endgroup::\n" ; +done + +# cleaning up and reporting +report_summary + +cleanup || rm -f ${LOCK_FILE} 2>/dev/null || : ; + +# unset when done +unset _TEST_ROOT_DIR 2>/dev/null || : ; +unset CODESPELL_OPTIONS 2>/dev/null || : ; + wait ; -exit ${EXIT_CODE:255} ; +exit ${EXIT_CODE:-255} ; diff --git a/tests/context.py b/tests/context.py index 3552f41..bf4b164 100644 --- a/tests/context.py +++ b/tests/context.py @@ -21,18 +21,58 @@ """This is pythonrepo testing module Template.""" +__module__ = """tests""" + +__name__ = """tests.context""" # skipcq: PYL-W0622 + +__doc__ = """ + + Robust imports: These statements import the entire "pythonrepo" module, + allowing access to all its functionalities within the test environment. + This can be flagged as an intentional + [cyclic-import](https://pylint.pycqa.org/en/latest/user_guide/messages/refactor/cyclic-import.html) + warning. + + Context for Testing. + + Meta Tests - Fixtures: + + Test fixtures by importing test context. + + >>> import tests.context as context + >>> + + >>> from context import pythonrepo as _pythonrepo + >>> + + >>> from context import profiling as _profiling + >>> + +""" + try: import sys - import os - if 'pythonrepo' in __file__: - sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -except Exception as badErr: - baton = ImportError(badErr, str("[CWE-758] Test module failed completely.")) - baton.module = __module__ - baton.path = __file__ - baton.__cause__ = badErr - raise baton + if not hasattr(sys, 'modules') or not sys.modules: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] sys.modules is not available or empty.") from None +except ImportError as err: + raise ImportError("[CWE-440] Unable to import sys module.") from err + +try: + if 'os' not in sys.modules: + import os + else: # pragma: no branch + os = sys.modules["""os"""] +except ImportError as err: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] OS Failed to import.") from err + +try: + if 'tests.profiling' not in sys.modules: + import tests.profiling as profiling + else: # pragma: no branch + profiling = sys.modules["""tests.profiling"""] +except ImportError as err: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] profiling Failed to import.") from err try: import pythonrepo as pythonrepo # skipcq: PYL-C0414 diff --git a/tests/fetch-test-reporter b/tests/fetch-test-reporter new file mode 120000 index 0000000..b718172 --- /dev/null +++ b/tests/fetch-test-reporter @@ -0,0 +1 @@ +../includes/fetch-test-reporter/fetch-test-reporter \ No newline at end of file diff --git a/tests/test_usage.py b/tests/test_usage.py index b75d253..9af1a7e 100644 --- a/tests/test_usage.py +++ b/tests/test_usage.py @@ -113,7 +113,7 @@ def checkPythonErrors(args=None, stderr=None): taintArgs = buildPythonCommand(args) theOutput = subprocess.check_output(taintArgs, stderr=stderr) if isinstance(theOutput, bytes): - # default to utf8 your milage may vary + # default to utf8 your mileage may vary theOutput = theOutput.decode('utf8') except Exception as err: theOutput = None