From 11c7672b41c231acd3a4aed7a393bb755d95ee3c Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Sat, 19 Mar 2022 11:01:51 +0100 Subject: [PATCH 1/3] Update package build tools --- .editorconfig | 2 +- .github/workflows/ci.yml | 24 +++++------- .github/workflows/release.yml | 11 ++---- .gitignore | 1 + linter-requirements.txt | 2 +- pyproject.toml | 71 +++++++++++++++++++++++++++++++++ s3file/__init__.py | 12 +++++- setup.cfg | 74 ----------------------------------- setup.py | 4 -- tests/conftest.py | 4 +- 10 files changed, 101 insertions(+), 104 deletions(-) create mode 100644 pyproject.toml delete mode 100755 setup.py diff --git a/.editorconfig b/.editorconfig index 788f064..ef5f8a5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,7 +11,7 @@ charset = utf-8 end_of_line = lf max_line_length = 88 -[*.{json,yml,yaml,js,jsx}] +[*.{json,yml,yaml,js,jsx,toml}] indent_size = 2 [LICENSE] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7160444..3b676de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/setup-python@v3 - - run: python -m pip install --upgrade pip setuptools wheel twine readme-renderer - uses: actions/checkout@v3 - - run: python setup.py sdist bdist_wheel + - run: python -m pip install --upgrade pip build wheel twine + - run: python -m build --sdist --wheel - run: python -m twine check dist/* standardjs: @@ -66,13 +66,11 @@ jobs: strategy: matrix: python-version: - - "3.7" - - "3.8" - "3.9" + - "3.10" django-version: - - "2.2" - - "3.1" - "3.2" + - "4.0" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -89,12 +87,10 @@ jobs: mkdir bin curl -qO "https://chromedriver.storage.googleapis.com/$(curl -q https://chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip" unzip chromedriver_linux64.zip -d bin - - name: Upgrade Python setuptools - run: python -m pip install --upgrade pip setuptools wheel codecov - - name: Install Django version ${{ matrix.django-version }} - run: python -m pip install django~=${{ matrix.django-version }} - - name: Run tests - run: PATH=$PATH:$(pwd)/bin python setup.py test - - run: codecov + + - run: python -m pip install .[test] codecov + - run: python -m pip install django~=${{ matrix.django-version }} + - run: python -m pytest env: - CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}} + PATH: $PATH:$(pwd)/bin + - run: codecov diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 50fde82..9a969f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,17 +11,14 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel twine + - run: python -m pip install --upgrade pip build wheel twine - uses: actions/setup-node@v3 - name: Install Node dependencies run: npm ci - name: Minify JavaScript files - run: npm run-script minify - - name: Build dist packages - run: python setup.py sdist bdist_wheel - - name: Upload packages - run: python -m twine upload dist/* + run: npm run minify + - run: python -m build --sdist --wheel + - run: python -m twine upload dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} diff --git a/.gitignore b/.gitignore index b106a28..cfb6375 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ var/ *.egg-info/ .installed.cfg *.egg +_version.py # PyInstaller # Usually these files are written by a python script from a template diff --git a/linter-requirements.txt b/linter-requirements.txt index a444077..2a8fb2f 100644 --- a/linter-requirements.txt +++ b/linter-requirements.txt @@ -2,4 +2,4 @@ bandit==1.7.4 black==22.1.0 flake8==4.0.1 isort==5.10.1 -pydocstyle==6.1.1 +pydocstyle[toml]==6.1.1 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9be4818 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[build-system] +requires = ["flit_core>=3.2", "flit_scm", "wheel"] +build-backend = "flit_scm:buildapi" + +[project] +name = "django-s3file" +authors = [ + { name = "Johannes Maron", email = "johannes@maron.family" } +] +readme = "README.rst" +license = { file = "LICENSE" } +dynamic = ["version", "description"] +classifiers = [ + "Development Status :: 6 - Mature", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: JavaScript", + "Topic :: Software Development", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Framework :: Django", + "Framework :: Django :: 3.2", + "Framework :: Django :: 4.0", +] +requires-python = ">=3.9" +dependencies = [ + "django>=2.0", + "django-storages", + "boto3", +] + +[project.optional-dependencies] +test = [ + "pytest >=2.7.3", + "pytest-cov", + "pytest-django", + "selenium", +] + +[project.urls] +Project-URL = "https://github.com/codingjoe/django-s3file" + +[tool.flit.module] +name = "s3file" + +[tool.setuptools_scm] +write_to = "s3file/_version.py" + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "--cov=s3file --tb=short -rxs" +testpaths = [ + "tests", +] +DJANGO_SETTINGS_MODULE = "tests.testapp.settings" + +[tool.isort] +atomic = true +line_length = 88 +known_first_party = "s3file, tests" +include_trailing_comma = true +default_section = "THIRDPARTY" +combine_as_imports = true + +[tool.pydocstyle] +add_ignore = "D1" diff --git a/s3file/__init__.py b/s3file/__init__.py index 40b3e5d..94bb8d5 100644 --- a/s3file/__init__.py +++ b/s3file/__init__.py @@ -1 +1,11 @@ -default_app_config = "s3file.apps.S3FileConfig" +"""A lightweight file uploader input for Django and Amazon S3.""" + +import django + +from . import _version + +__version__ = _version.version +VERSION = _version.version_tuple + +if django.VERSION < (4, 0): + default_app_config = "s3file.apps.S3FileConfig" diff --git a/setup.cfg b/setup.cfg index 46f9260..6f60592 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,78 +1,4 @@ -[metadata] -name = django-s3file -author = Johannes Hoppe -author_email = info@johanneshoppe.com -description = A lightweight file uploader input for Django and Amazon S3 -long_description = file: README.rst -long_description_content_type = text/x-rst -url = https://github.com/codingjoe/django-s3file -license = MIT -license_files = LICENSE -classifier = - Development Status :: 6 - Mature - Environment :: Web Environment - Framework :: Django - Intended Audience :: Developers - License :: OSI Approved :: MIT License - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: JavaScript - Topic :: Software Development - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Framework :: Django - Framework :: Django :: 2.2 - Framework :: Django :: 3.1 - -[options] -include_package_data = True -packages = find: -install_requires = - django>=2.0 - django-storages - boto3 -setup_requires = - setuptools_scm - pytest-runner -tests_require = - pytest - pytest-cov - pytest-django - selenium - -[options.package_data] -* = *.txt, *.rst, *.html, *.po - -[options.packages.find] -exclude = - tests - -[bdist_wheel] -universal = 1 - -[aliases] -test = pytest - -[tool:pytest] -addopts = --cov=s3file --cov-report xml --tb=short -rxs -DJANGO_SETTINGS_MODULE=tests.testapp.settings - [flake8] max-line-length=88 select = C,E,F,W,B,B950 ignore = E203, E501, W503, E731 - -[pydocstyle] -add_ignore = D1 - -[isort] -atomic = true -line_length = 88 -known_first_party = s3file, tests -include_trailing_comma = True -default_section=THIRDPARTY -combine_as_imports = true diff --git a/setup.py b/setup.py deleted file mode 100755 index 04b296e..0000000 --- a/setup.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - -setup(name="django-s3file", use_scm_version=True) diff --git a/tests/conftest.py b/tests/conftest.py index 439783d..4bfefd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest from django.core.files.base import ContentFile -from django.utils.encoding import force_text +from django.utils.encoding import force_str from selenium import webdriver from selenium.common.exceptions import WebDriverException @@ -15,7 +15,7 @@ def driver(): try: b = webdriver.Chrome(options=chrome_options) except WebDriverException as e: - pytest.skip(force_text(e)) + pytest.skip(force_str(e)) else: yield b b.quit() From a1b969a70c520704651750cee459e5097d9d7e2c Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Sat, 19 Mar 2022 12:12:46 +0100 Subject: [PATCH 2/3] Fix failing tests --- s3file/middleware.py | 6 +++++- s3file/storages.py | 7 ++++--- tests/test_forms.py | 9 ++++----- tests/test_middleware.py | 2 +- tests/testapp/settings.py | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/s3file/middleware.py b/s3file/middleware.py index 275f170..6c253ef 100644 --- a/s3file/middleware.py +++ b/s3file/middleware.py @@ -29,7 +29,11 @@ def get_files_from_storage(paths): for path in paths: path = pathlib.PurePosixPath(path) try: - f = storage.open(str(path.relative_to(storage.location))) + location = storage.aws_location + except AttributeError: + location = storage.location + try: + f = storage.open(str(path.relative_to(location))) f.name = path.name yield f except (OSError, ValueError): diff --git a/s3file/storages.py b/s3file/storages.py index 899a5d8..fa0b69b 100644 --- a/s3file/storages.py +++ b/s3file/storages.py @@ -11,11 +11,12 @@ class S3MockStorage(FileSystemStorage): @property - def location(self): + def aws_location(self): return getattr(settings, "AWS_LOCATION", "") - def path(self, name): - return safe_join(os.path.abspath(self.base_location), self.location, name) + @property + def location(self): + return safe_join(os.path.abspath(self.base_location), self.aws_location) class connection: class meta: diff --git a/tests/test_forms.py b/tests/test_forms.py index 99c72f8..ec1a9c4 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -36,10 +36,11 @@ def freeze(self, monkeypatch): """Freeze datetime and UUID.""" monkeypatch.setattr( "s3file.forms.S3FileInputMixin.upload_folder", - os.path.join(storage.location, "tmp"), + os.path.join(storage.aws_location, "tmp"), ) def test_value_from_datadict(self, client, upload_file): + print(storage.location) with open(upload_file) as f: uploaded_file = storage.save("test.jpg", f) response = client.post( @@ -227,7 +228,5 @@ def test_media(self): assert ClearableFileInput().media._js == ["s3file/js/s3file.js"] def test_upload_folder(self): - assert ClearableFileInput().upload_folder.startswith( - "custom/location/tmp/s3file/" - ) - assert len(ClearableFileInput().upload_folder) == 49 + assert "custom/location/tmp/s3file/" in ClearableFileInput().upload_folder + assert len(os.path.basename(ClearableFileInput().upload_folder)) == 22 diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 5225014..9dc552d 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -14,7 +14,7 @@ def test_get_files_from_storage(self): "tmp/s3file/test_get_files_from_storage", ContentFile(content) ) files = S3FileMiddleware.get_files_from_storage( - [os.path.join(storage.location, name)] + [os.path.join(storage.aws_location, name)] ) file = next(files) assert file.read() == content diff --git a/tests/testapp/settings.py b/tests/testapp/settings.py index eb19506..345369f 100644 --- a/tests/testapp/settings.py +++ b/tests/testapp/settings.py @@ -44,7 +44,7 @@ }, ] -USE_L10N = True +USE_TZ = True AWS_ACCESS_KEY_ID = "testaccessid" AWS_SECRET_ACCESS_KEY = "supersecretkey" From 1ead8ec2c6ab638e51e08320074f9570b52148dc Mon Sep 17 00:00:00 2001 From: Johannes Maron Date: Sat, 19 Mar 2022 12:46:51 +0100 Subject: [PATCH 3/3] Use Markdown readme instead or RST to use native mermaid diagrams --- Makefile | 2 - README.md | 231 +++++++++++++++++++++++++++++++++++++ README.rst | 217 ---------------------------------- http-message-flow.sequence | 8 -- http-message-flow.svg | 1 - pyproject.toml | 2 +- 6 files changed, 232 insertions(+), 229 deletions(-) delete mode 100644 Makefile create mode 100644 README.md delete mode 100644 README.rst delete mode 100644 http-message-flow.sequence delete mode 100644 http-message-flow.svg diff --git a/Makefile b/Makefile deleted file mode 100644 index 07fd945..0000000 --- a/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -http-message-flow.svg: http-message-flow.sequence - diagrams sequence http-message-flow.sequence http-message-flow.svg diff --git a/README.md b/README.md new file mode 100644 index 0000000..c274340 --- /dev/null +++ b/README.md @@ -0,0 +1,231 @@ +# django-s3file + +A lightweight file upload input for Django and Amazon S3. + +Django-S3File allows you to upload files directly AWS S3 effectively +bypassing your application server. This allows you to avoid long running +requests from large file uploads. This is particularly helpful for if +you run your service on AWS Lambda or Heroku where you have a hard +request limit. + +[![PyPi +Version](https://img.shields.io/pypi/v/django-s3file.svg)](https://pypi.python.org/pypi/django-s3file/) +[![Test +Coverage](https://codecov.io/gh/codingjoe/django-s3file/branch/master/graph/badge.svg)](https://codecov.io/gh/codingjoe/django-s3file) +[![GitHub +license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/codingjoe/django-s3file/master/LICENSE) + +## Features + +- lightweight: less 200 lines +- no JavaScript or Python dependencies (no jQuery) +- easy integration +- works just like the built-in +- extendable JavaScript API + +## For the Nerds + +```mermaid +sequenceDiagram + autonumber + actor Browser + Browser->>S3: POST large file + activate S3 + S3->>Browser: RESPONSE AWS S3 key + Browser->>Middleware: POST AWS S3 key + activate Middleware + Middleware->>S3: GET AWS S3 key + S3->>Middleware: RESPONSE large file promise + deactivate S3 + Middleware->>Django: request incl. large file promise + deactivate Middleware + activate Django + opt only if files is procssed by Django + Django-->>S3: GET large file + activate S3 + S3-->>Django: RESPONSE large file + deactivate S3 + end + Django->>Browser: RESPONSE success + deactivate Django +``` + +In a nutshell, we can bypass Django completely and have AWS handle +the upload or any processing. Of course, if you want to do something +with your file in Django, you can do so, just like before, with the +added advantage, that your file is served from within your datacenter. + +## Installation + +Make sure you have [Amazon S3 +storage](http://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html) +setup correctly. + +Just install S3file using `pip`. + +```bash +pip install django-s3file +# or +pipenv install django-s3file +``` + +Add the S3File app and middleware in your settings: + +```python +# settings.py + +INSTALLED_APPS = ( + '...', + 's3file', + '...', +) + +MIDDLEWARE = ( + '...', + 's3file.middleware.S3FileMiddleware', + '...', +) +``` + +## Usage + +S3File automatically replaces Django's `ClearableFileInput` widget, you +do not need to alter your code at all. + +The `ClearableFileInput` widget is only than automatically replaced when +the `DEFAULT_FILE_STORAGE` setting is set to `django-storages`' +`S3Boto3Storage` or the dummy `FileSystemStorage` is enabled. + +### Setting up the AWS S3 bucket + +#### Upload folder + +S3File uploads to a single folder. Files are later moved by Django when +they are saved to the `upload_to` location. + +It is recommended to [setup +expiration](http://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html) +for that folder, to ensure that old and unused file uploads don't add up +and produce costs. + +The default folder name is: `tmp/s3file` You can change it by changing +the `S3FILE_UPLOAD_PATH` setting. + +#### CORS policy + +You will need to allow `POST` from all origins. Just add the following +to your CORS policy. + +```json +[ + { + "AllowedHeaders": [ + "*" + ], + "AllowedMethods": [ + "POST" + ], + "AllowedOrigins": [ + "*" + ], + "ExposeHeaders": [], + "MaxAgeSeconds": 3000 + } +] +``` + +### Progress Bar + +S3File does emit progress signals that can be used to display some kind +of progress bar. Signals named `progress` are emitted for both each +individual file input as well as for the form as a whole. + +The progress signal carries the following details: + +```javascript +console.log(event.detail) + +{ + progress: 0.4725307607171312 // total upload progress of either a form or single input + loaded: 1048576 // total upload progress of either a form or single input + total: 2219064 // total bytes to upload + currentFile: File {…} // file object + currentFileName: "text.txt" // file name of the file currently uploaded + currentFileProgress: 0.47227834703299176 // upload progress of that file + originalEvent: ProgressEvent {…} // the original XHR onprogress event +} +``` + +The following example implements a Boostrap progress bar for upload +progress of an entire form. + +```html +
+
0%
+
+``` + +```javascript +(function () { + var form = document.getElementsByTagName('form')[0] + var progressBar = document.getElementsByClassName('progress-bar')[0] + + form.addEventListener('progress', function (event) { + // event.detail.progress is a value between 0 and 1 + var percent = Math.round(event.detail.progress * 100) + + progressBar.setAttribute('style', 'width:' + percent + '%') + progressBar.setAttribute('aria-valuenow', percent) + progressBar.innerText = percent + '%' + }) +})() +``` + +### Using S3File in development + +Using S3File in development can be helpful especially if you want to use +the progress signals described above. Therefore, S3File comes with a AWS +S3 dummy backend. It behaves similar to the real S3 storage backend. It +is automatically enabled, if the `DEFAULT_FILE_STORAGE` setting is set +to `FileSystemStorage`. + +To prevent users from accidentally using the `FileSystemStorage` and the +insecure S3 dummy backend in production, there is also an additional +deployment check that will error if you run Django\'s deployment check +suite: + +```shell +python manage.py check --deploy +``` + +We recommend always running the deployment check suite as part of your +deployment pipeline. + +### Uploading multiple files + +Django does have limited support for [uploading multiple +files](https://docs.djangoproject.com/en/stable/topics/http/file-uploads/#uploading-multiple-files). +S3File fully supports this feature. The custom middleware makes ensure +that files are accessible via `request.FILES`, even though they have +been uploaded to AWS S3 directly and not to your Django application +server. + +### Using optimized S3Boto3Storage + +Since `S3Boto3Storage` supports storing data from any other fileobj, it +uses a generalized `_save` function. This leads to the frontend +uploading the file to S3 and then copying it byte-by-byte to perform a +move operation just to rename the uploaded object. For large files this +leads to additional loading times for the user. + +That\'s why S3File provides an optimized version of this method at +`storages_optimized.S3OptimizedUploadStorage`. It uses the more +efficient `copy` method from S3, given that we know that we only copy +from one S3 location to another. + +```python +from s3file.storages_optimized import S3OptimizedUploadStorage + +class MyStorage(S3OptimizedUploadStorage): # Subclass and use like any other storage + default_acl = 'private' +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index d2d2211..0000000 --- a/README.rst +++ /dev/null @@ -1,217 +0,0 @@ -============= -django-s3file -============= - -A lightweight file upload input for Django and Amazon S3. - -Django-S3File allows you to upload files directly AWS S3 effectively -bypassing your application server. This allows you to avoid long running -requests from large file uploads. This is particularly helpful for if -you run your service on AWS Lambda or Heroku where you have a hard request -limit. - -|PyPi Version| |Test Coverage| |GitHub license| - --------- -Features --------- - -- lightweight: less 200 lines -- no JavaScript or Python dependencies (no jQuery) -- easy integration -- works just like the built-in -- extendable JavaScript API - -------------- -For the Nerds -------------- - -.. image:: http-message-flow.svg - ------------- -Installation ------------- - -Make sure you have `Amazon S3 storage`_ setup correctly. - -Just install S3file using ``pip``. - -.. code:: bash - - pip install django-s3file - # or - pipenv install django-s3file - -Add the S3File app and middleware in your settings: - -.. code:: python - - - INSTALLED_APPS = ( - '...', - 's3file', - '...', - ) - - MIDDLEWARE = ( - '...', - 's3file.middleware.S3FileMiddleware', - '...', - ) - ------ -Usage ------ - -S3File automatically replaces Django’s ``ClearableFileInput`` widget, -you do not need to alter your code at all. - -The ``ClearableFileInput`` widget is only than automatically replaced -when the ``DEFAULT_FILE_STORAGE`` setting is set to -``django-storages``\ ’ ``S3Boto3Storage`` or the dummy ``FileSystemStorage`` -is enabled. - -Setting up the AWS S3 bucket ----------------------------- - -Upload folder -~~~~~~~~~~~~~ - -S3File uploads to a single folder. Files are later moved by Django when -they are saved to the ``upload_to`` location. - -It is recommended to `setup expiration`_ for that folder, to ensure that -old and unused file uploads don’t add up and produce costs. - -The default folder name is: ``tmp/s3file`` You can change it by changing -the ``S3FILE_UPLOAD_PATH`` setting. - -CORS policy -~~~~~~~~~~~ - -You will need to allow ``POST`` from all origins. Just add the following -to your CORS policy. - -.. code:: json - - [ - { - "AllowedHeaders": [ - "*" - ], - "AllowedMethods": [ - "POST" - ], - "AllowedOrigins": [ - "*" - ], - "ExposeHeaders": [], - "MaxAgeSeconds": 3000 - } - ] - -Progress Bar ------------- - -S3File does emit progress signals that can be used to display some kind of progress bar. -Signals named ``progress`` are emitted for both each individual file input as well as -for the form as a whole. - -The progress signal carries the following details: - -.. code:: javascript - - console.log(event.detail) - - { - progress: 0.4725307607171312 // total upload progress of either a form or single input - loaded: 1048576 // total upload progress of either a form or single input - total: 2219064 // total bytes to upload - currentFile: File {…} // file object - currentFileName: "text.txt" // file name of the file currently uploaded - currentFileProgress: 0.47227834703299176 // upload progress of that file - originalEvent: ProgressEvent {…} // the original XHR onprogress event - } - - -The following example implements a Boostrap progress bar for upload progress of an -entire form. - -.. code:: html - -
-
0%
-
- -.. code:: javascript - - (function () { - var form = document.getElementsByTagName('form')[0] - var progressBar = document.getElementsByClassName('progress-bar')[0] - - form.addEventListener('progress', function (event) { - // event.detail.progress is a value between 0 and 1 - var percent = Math.round(event.detail.progress * 100) - - progressBar.setAttribute('style', 'width:' + percent + '%') - progressBar.setAttribute('aria-valuenow', percent) - progressBar.innerText = percent + '%' - }) - })() - - -Using S3File in development ---------------------------- - -Using S3File in development can be helpful especially if you want to use the progress -signals described above. Therefore, S3File comes with a AWS S3 dummy backend. -It behaves similar to the real S3 storage backend. It is automatically enabled, if the -``DEFAULT_FILE_STORAGE`` setting is set to ``FileSystemStorage``. - -To prevent users from accidentally using the ``FileSystemStorage`` and the insecure S3 -dummy backend in production, there is also an additional deployment check that will -error if you run Django's deployment check suite:: - - python manage.py check --deploy - -We recommend always running the deployment check suite as part of your deployment -pipeline. - -Uploading multiple files ------------------------- - -Django does have limited support for `uploading multiple files`_. S3File -fully supports this feature. The custom middleware makes ensure that -files are accessible via ``request.FILES``, even though they have been -uploaded to AWS S3 directly and not to your Django application server. - -.. _Amazon S3 storage: http://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html -.. _setup expiration: http://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html -.. _uploading multiple files: https://docs.djangoproject.com/en/stable/topics/http/file-uploads/#uploading-multiple-files - -.. |PyPi Version| image:: https://img.shields.io/pypi/v/django-s3file.svg - :target: https://pypi.python.org/pypi/django-s3file/ -.. |Test Coverage| image:: https://codecov.io/gh/codingjoe/django-s3file/branch/master/graph/badge.svg - :target: https://codecov.io/gh/codingjoe/django-s3file -.. |GitHub license| image:: https://img.shields.io/badge/license-MIT-blue.svg - :target: https://raw.githubusercontent.com/codingjoe/django-s3file/master/LICENSE - -Using optimized S3Boto3Storage ------------------------------- - -Since ``S3Boto3Storage`` supports storing data from any other fileobj, -it uses a generalized ``_save`` function. This leads to the frontend uploading -the file to S3 and then copying it byte-by-byte to perform a move operation just -to rename the uploaded object. For large files this leads to additional loading -times for the user. - -That's why S3File provides an optimized version of this method at -``storages_optimized.S3OptimizedUploadStorage``. It uses the more efficient -``copy`` method from S3, given that we know that we only copy from one S3 location to another. - -.. code:: python - - from s3file.storages_optimized import S3OptimizedUploadStorage - - class MyStorage(S3OptimizedUploadStorage): # Subclass and use like any other storage - default_acl = 'private' diff --git a/http-message-flow.sequence b/http-message-flow.sequence deleted file mode 100644 index 1365113..0000000 --- a/http-message-flow.sequence +++ /dev/null @@ -1,8 +0,0 @@ -Client->AWS S3:POST large file -AWS S3->Client:RESPONSE AWS S3 key -Client->S3FileMiddleware:POST AWS S3 key -S3FileMiddleware->AWS S3:GET AWS S3 key -AWS S3->S3FileMiddleware:RESPONSE large file -S3FileMiddleware->DjangoView:request incl. large file -DjangoView->Client:RESPONSE success - diff --git a/http-message-flow.svg b/http-message-flow.svg deleted file mode 100644 index 4914b4e..0000000 --- a/http-message-flow.svg +++ /dev/null @@ -1 +0,0 @@ -Created with Raphaël 2.1.4ClientClientAWS S3AWS S3S3FileMiddlewareS3FileMiddlewareDjangoViewDjangoViewPOST large fileRESPONSE AWS S3 keyPOST AWS S3 keyGET AWS S3 keyRESPONSE large filerequest incl. large fileRESPONSE success \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9be4818..ada07ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "django-s3file" authors = [ { name = "Johannes Maron", email = "johannes@maron.family" } ] -readme = "README.rst" +readme = "README.md" license = { file = "LICENSE" } dynamic = ["version", "description"] classifiers = [