commit c40ee0a6bf349540528180363f37f8bc1046eb3f Author: Remilia Da Costa Faro Date: Tue Sep 10 11:23:04 2024 +0200 Initial commit! diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..682995b --- /dev/null +++ b/.gitignore @@ -0,0 +1,141 @@ +local_settings.py +.DS_Store +deploy.sh + +*.log +*.pot +*.pyc +__pycache__ +db.sqlite3 +media + +# Backup files # +*.bak + +# If you are using PyCharm # +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# File-based project format +*.iws + +# IntelliJ +out/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Python # +*.py[cod] +*$py.class + +# Distribution / packaging +.Python build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.whl +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Sublime Text # +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# sftp configuration file +sftp-config.json + +# Package control specific files Package +Control.last-run +Control.ca-list +Control.ca-bundle +Control.system-ca-bundle +GitHub.sublime-settings + +# Visual Studio Code # +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 0000000..b0fb598 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,26 @@ + + + + + sqlite.xerial + true + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/db.sqlite3 + $ProjectFileDir$ + + + file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.43.0/org/xerial/sqlite-jdbc/3.43.0.0/sqlite-jdbc-3.43.0.0.jar + + + + + sqlite.xerial + true + true + $PROJECT_DIR$/kakigoori/settings.py + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/db.sqlite3 + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..1d86f38 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,21 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/kakigoori.iml b/.idea/kakigoori.iml new file mode 100644 index 0000000..e66e84a --- /dev/null +++ b/.idea/kakigoori.iml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..67c7da3 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..40b044e --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/ruff.xml b/.idea/ruff.xml new file mode 100644 index 0000000..bae7814 --- /dev/null +++ b/.idea/ruff.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml new file mode 100644 index 0000000..c70980f --- /dev/null +++ b/.idea/watcherTasks.xml @@ -0,0 +1,25 @@ + + + + + + + + \ No newline at end of file diff --git a/images/__init__.py b/images/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/images/admin.py b/images/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/images/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/images/apps.py b/images/apps.py new file mode 100644 index 0000000..5f752ba --- /dev/null +++ b/images/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ImagesConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "images" diff --git a/images/decorators.py b/images/decorators.py new file mode 100644 index 0000000..e37a5b6 --- /dev/null +++ b/images/decorators.py @@ -0,0 +1,56 @@ +from functools import wraps +from typing import Optional + +from django.http import JsonResponse, HttpResponseForbidden + +from images.models import Image, AuthorizationKey + + +def get_image(func): + @wraps(func) + def wrapper(request, *args, **kwargs): + image_id = kwargs["image_id"] + del kwargs["image_id"] + image = Image.objects.filter(id=image_id).first() + if image is None: + return JsonResponse({"error": "Image not found"}, status=404) + + return func(request=request, image=image, *args, **kwargs) + + return wrapper + + +def can_upload_image(func): + @wraps(func) + def wrapper(request, *args, **kwargs): + if "Authorization" not in request.headers: + return HttpResponseForbidden() + + authorization_key: Optional[AuthorizationKey] = AuthorizationKey.objects.filter( + id=request.headers["Authorization"], can_upload_image=True + ).first() + + if not authorization_key: + return HttpResponseForbidden() + + return func(request, *args, **kwargs) + + return wrapper + + +def can_upload_variant(func): + @wraps(func) + def wrapper(request, *args, **kwargs): + if "Authorization" not in request.headers: + return HttpResponseForbidden() + + authorization_key: Optional[AuthorizationKey] = AuthorizationKey.objects.filter( + id=request.headers["Authorization"], can_upload_variant=True + ).first() + + if not authorization_key: + return HttpResponseForbidden() + + return func(request, *args, **kwargs) + + return wrapper diff --git a/images/management/__init__.py b/images/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/images/management/commands/__init__.py b/images/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/images/management/commands/create_tasks.py b/images/management/commands/create_tasks.py new file mode 100644 index 0000000..749f047 --- /dev/null +++ b/images/management/commands/create_tasks.py @@ -0,0 +1,35 @@ +from django.core.management.base import BaseCommand + +from images.models import Image, ImageVariant, ImageVariantTask + + +class Command(BaseCommand): + def handle(self, *args, **options): + images = Image.objects.filter(model_version=2).all() + images_len = len(images) + for index, image in enumerate(images): + print(f"Image {index}/{images_len}") + variants = ImageVariant.objects.filter(image=image).all() + variant_sizes = list(set(map(lambda x: (x.width, x.height), variants))) + + for image_type in ["avif", "webp", "jpegli"]: + for variant_size in variant_sizes: + avif_variant = [ + x + for x in variants + if x.width == variant_size[0] + and x.height == variant_size[1] + and x.file_type == image_type + ] + if not avif_variant: + ImageVariantTask( + image=image, + height=variant_size[1], + width=variant_size[0], + original_file_type=image.imagevariant_set.filter( + is_full_size=True, file_type__in=["jpg", "png"] + ) + .first() + .file_type, + file_type=image_type, + ).save() diff --git a/images/management/commands/upgrade_images.py b/images/management/commands/upgrade_images.py new file mode 100644 index 0000000..4e900ab --- /dev/null +++ b/images/management/commands/upgrade_images.py @@ -0,0 +1,215 @@ +import sys +from io import BytesIO +from PIL import Image as PILImage + + +import boto3 +from botocore.config import Config +from django.core.management.base import BaseCommand, CommandError + +from images.models import Image, ImageVariant + + +class Command(BaseCommand): + def get_b2_resource(self, endpoint, key_id, application_key): + b2 = boto3.resource( + service_name="s3", + endpoint_url=endpoint, + aws_access_key_id=key_id, + aws_secret_access_key=application_key, + config=Config(signature_version="s3v4"), + ) + + bucket = b2.Bucket("kakigoori") + + return bucket + + def handle(self, *args, **options): + images = Image.objects.filter(model_version=1).order_by("-creation_date").all() + images_len = len(images) + progress = 1 + print(f"{images_len} images found") + for image in images: + print(f"Image {progress}/{images_len}") + try: + print("Upgrading image %s" % image.id) + + bucket = self.get_b2_resource( + "https://s3.eu-central-003.backblazeb2.com", + "0032dcec6092f3e0000000021", + "K0039Xb1GE/An9P1ccK2B+19pXrKnnU", + ) + + print("Getting original image...") + + original_image = BytesIO() + bucket.download_fileobj( + f"{image.backblaze_filepath}/original.jpg", original_image + ) + original_image.seek(0) + + with PILImage.open(original_image) as im: + image.height = im.height + image.width = im.width + image.save() + + print("Copying original images...") + + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/original.jpg", + }, + f"{image.backblaze_filepath}/{image.width}-{image.height}/image.jpg", + ) + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/thumbnail.jpg", + }, + f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/image.jpg", + ) + + delete_objects_list = [f"{image.backblaze_filepath}/thumbnail.jpg"] + + ImageVariant( + image=image, + height=image.height, + width=image.width, + is_full_size=True, + file_type="jpg", + ).save() + ImageVariant( + image=image, + height=image.thumbnail_size[1], + width=image.thumbnail_size[0], + is_full_size=False, + file_type="jpg", + ).save() + + if image.is_jpegli_available: + print("Copying JPEGLI images...") + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/jpegli.jpg", + }, + f"{image.backblaze_filepath}/{image.width}-{image.height}/jpegli.jpg", + ) + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/thumbnail_jpegli.jpg", + }, + f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/jpegli.jpg", + ) + + ImageVariant( + image=image, + height=image.height, + width=image.width, + is_full_size=True, + file_type="jpegli", + ).save() + ImageVariant( + image=image, + height=image.thumbnail_size[0], + width=image.thumbnail_size[1], + is_full_size=False, + file_type="jpegli", + ).save() + + delete_objects_list += [ + f"{image.backblaze_filepath}/jpegli.jpg", + f"{image.backblaze_filepath}/thumbnail_jpegli.jpg", + ] + if image.is_avif_available: + print("Copying AVIF images...") + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/optimized.avif", + }, + f"{image.backblaze_filepath}/{image.width}-{image.height}/image.avif", + ) + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/thumbnail.avif", + }, + f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/image.avif", + ) + + ImageVariant( + image=image, + height=image.height, + width=image.width, + is_full_size=True, + file_type="avif", + ).save() + ImageVariant( + image=image, + height=image.thumbnail_size[0], + width=image.thumbnail_size[1], + is_full_size=False, + file_type="avif", + ).save() + + delete_objects_list += [ + f"{image.backblaze_filepath}/optimized.avif", + f"{image.backblaze_filepath}/thumbnail.avif", + ] + if image.is_webp_available: + print("Copying WebP images...") + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/optimized.webp", + }, + f"{image.backblaze_filepath}/{image.width}-{image.height}/image.webp", + ) + bucket.copy( + { + "Bucket": "kakigoori", + "Key": f"{image.backblaze_filepath}/thumbnail.webp", + }, + f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/image.webp", + ) + + ImageVariant( + image=image, + height=image.height, + width=image.width, + is_full_size=True, + file_type="webp", + ).save() + ImageVariant( + image=image, + height=image.thumbnail_size[0], + width=image.thumbnail_size[1], + is_full_size=False, + file_type="webp", + ).save() + + delete_objects_list += [ + f"{image.backblaze_filepath}/optimized.webp", + f"{image.backblaze_filepath}/thumbnail.webp", + ] + + print("Saving...") + + image.model_version = 2 + image.save() + + print("Deleting old files...") + + bucket.delete_objects( + Delete={"Objects": [{"Key": x} for x in delete_objects_list]} + ) + + progress += 1 + except KeyboardInterrupt: + sys.exit(1) + except: + progress += 1 + pass diff --git a/images/migrations/0001_initial.py b/images/migrations/0001_initial.py new file mode 100644 index 0000000..199af39 --- /dev/null +++ b/images/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# Generated by Django 5.0 on 2023-12-08 09:24 + +import django.utils.timezone +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Image", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "creation_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("uploaded", models.BooleanField(default=False)), + ("original_name", models.CharField(max_length=150)), + ("original_mime_type", models.CharField(max_length=10)), + ("original_md5", models.CharField(max_length=32)), + ("is_mozjpeg_available", models.BooleanField(default=False)), + ("is_webp_available", models.BooleanField(default=False)), + ("is_avif_available", models.BooleanField(default=False)), + ], + ), + ] diff --git a/images/migrations/0002_remove_image_is_mozjpeg_available_and_more.py b/images/migrations/0002_remove_image_is_mozjpeg_available_and_more.py new file mode 100644 index 0000000..974fb9e --- /dev/null +++ b/images/migrations/0002_remove_image_is_mozjpeg_available_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0 on 2024-04-08 09:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='is_mozjpeg_available', + ), + migrations.AddField( + model_name='image', + name='is_jpegli_available', + field=models.BooleanField(default=False), + ), + ] diff --git a/images/migrations/0003_authorizationkeys_image_height_image_model_version_and_more.py b/images/migrations/0003_authorizationkeys_image_height_image_model_version_and_more.py new file mode 100644 index 0000000..b9debff --- /dev/null +++ b/images/migrations/0003_authorizationkeys_image_height_image_model_version_and_more.py @@ -0,0 +1,50 @@ +# Generated by Django 5.0 on 2024-06-22 13:56 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0002_remove_image_is_mozjpeg_available_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='AuthorizationKeys', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=150)), + ('can_upload', models.BooleanField(default=False)), + ('can_convert', models.BooleanField(default=False)), + ], + ), + migrations.AddField( + model_name='image', + name='height', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='image', + name='model_version', + field=models.IntegerField(default=1), + ), + migrations.AddField( + model_name='image', + name='width', + field=models.IntegerField(default=0), + ), + migrations.CreateModel( + name='ImageVariant', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('height', models.IntegerField()), + ('width', models.IntegerField()), + ('is_full_size', models.BooleanField(default=False)), + ('file_type', models.CharField(max_length=10)), + ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='images.image')), + ], + ), + ] diff --git a/images/migrations/0004_rename_authorizationkeys_authorizationkey_and_more.py b/images/migrations/0004_rename_authorizationkeys_authorizationkey_and_more.py new file mode 100644 index 0000000..aee18bd --- /dev/null +++ b/images/migrations/0004_rename_authorizationkeys_authorizationkey_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0 on 2024-06-25 09:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("images", "0003_authorizationkeys_image_height_image_model_version_and_more"), + ] + + operations = [ + migrations.RenameModel( + old_name="AuthorizationKeys", + new_name="AuthorizationKey", + ), + migrations.RenameField( + model_name="authorizationkey", + old_name="can_convert", + new_name="can_upload_variant", + ), + ] diff --git a/images/migrations/0005_rename_can_upload_authorizationkey_can_upload_image.py b/images/migrations/0005_rename_can_upload_authorizationkey_can_upload_image.py new file mode 100644 index 0000000..575092c --- /dev/null +++ b/images/migrations/0005_rename_can_upload_authorizationkey_can_upload_image.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0 on 2024-06-25 09:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("images", "0004_rename_authorizationkeys_authorizationkey_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="authorizationkey", + old_name="can_upload", + new_name="can_upload_image", + ), + ] diff --git a/images/migrations/0006_imagevarianttask.py b/images/migrations/0006_imagevarianttask.py new file mode 100644 index 0000000..0d425ed --- /dev/null +++ b/images/migrations/0006_imagevarianttask.py @@ -0,0 +1,40 @@ +# Generated by Django 5.0 on 2024-06-26 21:34 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("images", "0005_rename_can_upload_authorizationkey_can_upload_image"), + ] + + operations = [ + migrations.CreateModel( + name="ImageVariantTask", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("height", models.IntegerField()), + ("width", models.IntegerField()), + ("original_file_type", models.CharField(max_length=10)), + ("file_type", models.CharField(max_length=10)), + ( + "image", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="images.image" + ), + ), + ], + ), + ] diff --git a/images/migrations/__init__.py b/images/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/images/models.py b/images/models.py new file mode 100644 index 0000000..35508f9 --- /dev/null +++ b/images/models.py @@ -0,0 +1,138 @@ +import uuid +from io import BytesIO + +from django.db import models +from django.utils import timezone + +from images.utils import get_b2_resource +from PIL import Image as PILImage, ImageOps + + +class Image(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + creation_date = models.DateTimeField(default=timezone.now) + uploaded = models.BooleanField(default=False) + original_name = models.CharField(max_length=150) + original_mime_type = models.CharField(max_length=10) + original_md5 = models.CharField(max_length=32) + is_webp_available = models.BooleanField(default=False) + is_avif_available = models.BooleanField(default=False) + is_jpegli_available = models.BooleanField(default=False) + model_version = models.IntegerField(default=1) + height = models.IntegerField(default=0) + width = models.IntegerField(default=0) + + @property + def thumbnail_size(self): + if self.height > self.width: + return int(600 * self.width / self.height), 600 + else: + return 600, int(600 * self.height / self.width) + + @property + def backblaze_filepath(self): + return f"{self.id.hex[:2]}/{self.id.hex[2:4]}/{self.id.hex}" + + def create_variant_tasks(self, width, height, original_file_type): + ImageVariantTask( + image=self, + height=height, + width=width, + original_file_type=original_file_type, + file_type="avif", + ).save() + + ImageVariantTask( + image=self, + height=height, + width=width, + original_file_type=original_file_type, + file_type="webp", + ).save() + + ImageVariantTask( + image=self, + height=height, + width=width, + original_file_type=original_file_type, + file_type="jpegli", + ).save() + + def create_variant(self, width, height): + bucket = get_b2_resource() + + original_image = BytesIO() + original_variant = self.imagevariant_set.filter( + is_full_size=True, file_type__in=["jpg", "png"] + ).first() + bucket.download_fileobj( + f"{self.backblaze_filepath}/{original_variant.width}-{original_variant.height}/image.{original_variant.file_type}", + original_image, + ) + original_image.seek(0) + + resized_image = BytesIO() + + file_extension = "jpg" + + with PILImage.open(original_image) as im: + ImageOps.exif_transpose(im, in_place=True) + im.thumbnail((width, height)) + + if im.has_transparency_data: + try: + im.save(resized_image, "PNG") + file_extension = "png" + except OSError: + im.convert("RGB").save(resized_image, "JPEG") + else: + try: + im.save(resized_image, "JPEG") + except OSError: + im.convert("RGB").save(resized_image, "JPEG") + + resized_image.seek(0) + + bucket.upload_fileobj( + resized_image, + f"{self.backblaze_filepath}/{width}-{height}/image.{file_extension}", + ) + + image_variant = ImageVariant( + image=self, + height=height, + width=width, + is_full_size=False, + file_type=file_extension, + ) + + image_variant.save() + + self.create_variant_tasks(width, height, file_extension) + + return image_variant + + +class ImageVariant(models.Model): + image = models.ForeignKey(Image, on_delete=models.CASCADE) + height = models.IntegerField() + width = models.IntegerField() + is_full_size = models.BooleanField(default=False) + file_type = models.CharField(max_length=10) + + +class ImageVariantTask(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + created_at = models.DateTimeField(auto_now_add=True) + image = models.ForeignKey(Image, on_delete=models.CASCADE) + height = models.IntegerField() + width = models.IntegerField() + original_file_type = models.CharField(max_length=10) + file_type = models.CharField(max_length=10) + + +class AuthorizationKey(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=150) + can_upload_image = models.BooleanField(default=False) + can_upload_variant = models.BooleanField(default=False) diff --git a/images/tests.py b/images/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/images/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/images/urls.py b/images/urls.py new file mode 100644 index 0000000..559144b --- /dev/null +++ b/images/urls.py @@ -0,0 +1,34 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path("", views.index, name="index"), + path("upload", views.upload, name="images.upload"), + path( + "conversion_tasks/upload_variant", + views.upload_variant, + name="images.upload_variant", + ), + path( + "conversion_tasks/", + views.image_type_optimization_needed, + name="images.avif_optimization_needed", + ), + path("/", views.get, name="images.get"), + path( + "//thumbnail", + views.get_thumbnail, + name="images.get_thumbnail", + ), + path( + "/height//", + views.get_image_with_height, + name="images.get_image_with_height", + ), + path( + "/width//", + views.get_image_with_width, + name="images.get_image_with_width", + ), +] diff --git a/images/utils.py b/images/utils.py new file mode 100644 index 0000000..804a4cc --- /dev/null +++ b/images/utils.py @@ -0,0 +1,17 @@ +import boto3 +from botocore.config import Config +from django.conf import settings + + +def get_b2_resource(): + b2 = boto3.resource( + service_name="s3", + endpoint_url=settings.S3_ENDPOINT, + aws_access_key_id=settings.S3_KEY_ID, + aws_secret_access_key=settings.S3_ACCESS_KEY, + config=Config(signature_version="s3v4"), + ) + + bucket = b2.Bucket(settings.S3_BUCKET) + + return bucket diff --git a/images/views.py b/images/views.py new file mode 100644 index 0000000..6b85dc1 --- /dev/null +++ b/images/views.py @@ -0,0 +1,241 @@ +import hashlib + +from PIL import Image as PILImage +from PIL import JpegImagePlugin +from django.conf import settings +from django.http import ( + JsonResponse, + HttpResponseBadRequest, + HttpResponseNotFound, +) +from django.shortcuts import redirect, render +from django.views.decorators.csrf import csrf_exempt + +from images.decorators import ( + get_image, + can_upload_variant, + can_upload_image, +) +from images.models import Image, ImageVariant, ImageVariantTask +from images.utils import get_b2_resource + +JpegImagePlugin._getmp = lambda x: None + + +def index(request): + return render(request, "index.html") + + +@csrf_exempt +@can_upload_image +def upload(request): + file = request.FILES["file"] + + bucket = get_b2_resource() + + file_md5_hash = hashlib.file_digest(file, "md5").hexdigest() + file.seek(0) + + with PILImage.open(file) as im: + height, width = (im.height, im.width) + + if im.format == "JPEG": + file_extension = "jpg" + elif im.format == "PNG": + file_extension = "png" + else: + return {"created": False, "error": "Uploaded file should be JPEG or PNG"} + + file.seek(0) + + same_md5_image = Image.objects.filter(original_md5=file_md5_hash).first() + if same_md5_image: + return {"created": False, "id": same_md5_image.id} + + image = Image( + original_name=file.name, + original_mime_type=file.content_type, + original_md5=file_md5_hash, + height=height, + width=width, + model_version=2, + ) + + image.save() + + bucket.upload_fileobj( + file, f"{image.backblaze_filepath}/{width}-{height}/image.{file_extension}" + ) + + ImageVariant.objects.get_or_create( + image=image, + height=height, + width=width, + file_type=file_extension, + is_full_size=True, + ) + + image.create_variant_tasks(image.width, image.height, file_extension) + image.uploaded = True + image.save() + + return JsonResponse({"created": True, "id": image.id}) + + +@can_upload_variant +def image_type_optimization_needed(request, image_type): + tasks = ImageVariantTask.objects.filter(file_type=image_type).all() + + return JsonResponse( + { + "variants": [ + { + "image_id": task.image.id, + "task_id": task.id, + "height": task.height, + "width": task.width, + "file_type": task.original_file_type, + } + for task in tasks + ] + } + ) + + +@csrf_exempt +@can_upload_variant +def upload_variant(request): + if "task_id" not in request.POST: + return HttpResponseBadRequest() + + task = ImageVariantTask.objects.filter(id=request.POST["task_id"]).first() + + if not task: + return HttpResponseNotFound() + + image = task.image + height = task.height + width = task.width + file_type = task.file_type + file = request.FILES["file"] + + if file_type == "jpegli": + file_name = "jpegli.jpg" + else: + file_name = "image." + file_type + + upload_path = f"{image.backblaze_filepath}/{width}-{height}/{file_name}" + + bucket = get_b2_resource() + + bucket.upload_fileobj(file, upload_path) + + ImageVariant.objects.get_or_create( + image=image, + height=height, + width=width, + file_type=file_type, + is_full_size=(height == image.height and width == image.width), + ) + + task.delete() + + return JsonResponse({"status": "ok"}) + + +def image_with_size(request, image, width, height, image_type): + image_variants = ImageVariant.objects.filter( + image=image, height=height, width=width + ) + if image_type != "auto": + if image_type == "original": + image_variants = image_variants.filter(file_type__in=["jpg", "png"]) + else: + image_variants = image_variants.filter(file_type=image_type) + + variants = image_variants.all() + + if not variants: + if image_type != "auto" and image_type != "original": + return JsonResponse({"error": "Image version not available"}, status=404) + else: + image_variant = image.create_variant(width, height) + + return redirect( + f"{settings.S3_PUBLIC_BASE_PATH}/{image.backblaze_filepath}/{width}-{height}/image.{image_variant.file_type}" + ) + + if image_type == "auto": + variants_preferred_order = ["avif", "webp", "jpegli", "jpg", "png"] + elif image_type == "original": + variants_preferred_order = ["jpg", "png"] + else: + variants_preferred_order = [image_type] + + accept_header = request.headers.get("Accept", default="") + + for file_type in variants_preferred_order: + if ( + file_type == "avif" + and image_type == "auto" + and "image/avif" not in accept_header + ): + continue + + if ( + file_type == "webp" + and image_type == "auto" + and "image/webp" not in accept_header + ): + continue + + variant = [x for x in image_variants if x.file_type == file_type] + if not variant: + continue + + variant = variant[0] + + if file_type == "jpegli": + file_name = "jpegli.jpg" + else: + file_name = "image." + file_type + + return redirect( + f"{settings.S3_PUBLIC_BASE_PATH}/{image.backblaze_filepath}/{variant.width}-{variant.height}/{file_name}" + ) + + return HttpResponseNotFound() + + +@get_image +def get_image_with_height(request, image, height, image_type): + if height >= image.height: + height = image.height + width = image.width + else: + width = int(height * image.width / image.height) + + return image_with_size(request, image, width, height, image_type) + + +@get_image +def get_image_with_width(request, image, width, image_type): + if width >= image.width: + width = image.width + height = image.height + else: + height = int(width * image.height / image.width) + + return image_with_size(request, image, width, height, image_type) + + +@get_image +def get(request, image, image_type): + return image_with_size(request, image, image.width, image.height, image_type) + + +@get_image +def get_thumbnail(request, image, image_type): + width, height = image.thumbnail_size + + return image_with_size(request, image, width, height, image_type) diff --git a/kakigoori/__init__.py b/kakigoori/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kakigoori/asgi.py b/kakigoori/asgi.py new file mode 100644 index 0000000..d2482d7 --- /dev/null +++ b/kakigoori/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for kakigoori project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kakigoori.settings") + +application = get_asgi_application() diff --git a/kakigoori/settings.py b/kakigoori/settings.py new file mode 100644 index 0000000..dbb3725 --- /dev/null +++ b/kakigoori/settings.py @@ -0,0 +1,119 @@ +""" +Django settings for kakigoori project. + +Generated by 'django-admin startproject' using Django 4.2.5. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "images.apps.ImagesConfig", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "kakigoori.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [BASE_DIR / "templates"], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "kakigoori.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +from .local_settings import * diff --git a/kakigoori/urls.py b/kakigoori/urls.py new file mode 100644 index 0000000..eac3b23 --- /dev/null +++ b/kakigoori/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for kakigoori project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +# from django.contrib import admin +from django.urls import path, include + +urlpatterns = [ + path("", include("images.urls")), +] diff --git a/kakigoori/wsgi.py b/kakigoori/wsgi.py new file mode 100644 index 0000000..fd97d4f --- /dev/null +++ b/kakigoori/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for kakigoori project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kakigoori.settings") + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..0ca06c4 --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kakigoori.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..68ed705 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +asgiref==3.7.2 +boto3==1.33.10 +botocore==1.33.10 +certifi==2023.11.17 +charset-normalizer==3.3.2 +Django==5.0 +gunicorn==21.2.0 +idna==3.6 +jmespath==1.0.1 +packaging==23.2 +Pillow==10.1.0 +python-dateutil==2.8.2 +requests==2.31.0 +s3transfer==0.8.2 +sentry-sdk==1.38.0 +six==1.16.0 +sqlparse==0.4.4 +urllib3==2.0.7 \ No newline at end of file diff --git a/static/css/index.css b/static/css/index.css new file mode 100644 index 0000000..4bc93b8 --- /dev/null +++ b/static/css/index.css @@ -0,0 +1,85 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; + background-color: #fdf6e3; + margin: 0; +} + +.container { + width: 95%; + margin: 0 auto; +} +@media (min-width: 576px) { + .container { + width: 540px; + } +} +@media (min-width: 768px) { + .container { + width: 720px; + } +} +@media (min-width: 992px) { + .container { + width: 960px; + } +} +@media (min-width: 1200px) { + .container { + width: 1140px; + } +} +@media (min-width: 1400px) { + .container { + width: 1320px; + } +} + +header { + display: flex; + justify-content: center; + align-items: center; + margin-top: 2rem; + gap: 2rem; + padding: 3rem 1rem; + margin-bottom: 3rem; + border-bottom: 1px solid rgba(0, 0, 0, 0.1); + flex-wrap: wrap; +} +header div:has(img) { + display: flex; + justify-content: center; + align-items: center; +} +header img { + width: 80%; + max-width: 300px; +} + +main { + margin-bottom: 5rem; +} +main #examples-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 1rem; + margin-bottom: 2rem; +} +main #examples-grid p > strong { + word-break: break-all; +} +@media (max-width: 768px) { + main #examples-grid { + grid-template-columns: 1fr; + } +} +main #examples-grid img { + max-width: 100%; +} +main details { + margin-bottom: 1rem; +} +main hr { + border: 1px solid rgba(0, 0, 0, 0.1); +} + +/*# sourceMappingURL=index.css.map */ diff --git a/static/css/index.css.map b/static/css/index.css.map new file mode 100644 index 0000000..2f83649 --- /dev/null +++ b/static/css/index.css.map @@ -0,0 +1 @@ +{"version":3,"sourceRoot":"","sources":["index.scss"],"names":[],"mappings":"AAAA;EACC;EACA;EACA;;;AAGD;EACC;EACA;;AAEA;EAJD;IAKE;;;AAGD;EARD;IASE;;;AAGD;EAZD;IAaE;;;AAGD;EAhBD;IAiBE;;;AAGD;EApBD;IAqBE;;;;AAIF;EACC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;AAEA;EACC;EACA;EACA;;AAGD;EACC;EACA;;;AAIF;EACC;;AAEA;EACC;EACA;EACA;EACA;;AAEA;EACC;;AAGD;EAVD;IAWE;;;AAGD;EACC;;AAIF;EACC;;AAGD;EACC","file":"index.css"} \ No newline at end of file diff --git a/static/css/index.scss b/static/css/index.scss new file mode 100644 index 0000000..96b588b --- /dev/null +++ b/static/css/index.scss @@ -0,0 +1,84 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; + background-color: #fdf6e3; + margin: 0; +} + +.container { + width: 95%; + margin: 0 auto; + + @media (min-width: 576px) { + width: 540px; + } + + @media (min-width: 768px) { + width: 720px; + } + + @media (min-width: 992px) { + width: 960px; + } + + @media (min-width: 1200px) { + width: 1140px; + } + + @media (min-width: 1400px) { + width: 1320px; + } +} + +header { + display: flex; + justify-content: center; + align-items: center; + margin-top: 2rem; + gap: 2rem; + padding: 3rem 1rem; + margin-bottom: 3rem; + border-bottom: 1px solid rgba(0, 0, 0, 0.1); + flex-wrap: wrap; + + div:has(img) { + display: flex; + justify-content: center; + align-items: center; + } + + img { + width: 80%; + max-width: 300px; + } +} + +main { + margin-bottom: 5rem; + + #examples-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 1rem; + margin-bottom: 2rem; + + p > strong { + word-break: break-all; + } + + @media (max-width: 768px) { + grid-template-columns: 1fr; + } + + img { + max-width: 100%; + } + } + + details { + margin-bottom: 1rem; + } + + hr { + border: 1px solid rgba(0, 0, 0, 0.1); + } +} \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..a58e40f --- /dev/null +++ b/templates/index.html @@ -0,0 +1,96 @@ +{% load static %} + + + + + + + Kakigoori + + + +
+
+ +
+
+

Kakigoori

+

Image distribution for the web, simplified

+

A project by Remilia Da Costa Faro

+
+
+ +
+
+

Upload an image once, use it everywhere on the web

+

+ Kakigoori is an image management service to publish images on the web. Upload it once, and Kakigoori will create versions of the image, optimized for the web (AVIF, WebP, Jpegli). You can also request for custom sized images, of which the JPG/PNG versions will be generated on the fly, and the other versions will be created within a few minutes. +

+

+ Kakigoori's images are stored by Backblaze and distributed by Cloudflare. No matter where you are in the world, images stored by Kakigoori will display as fast as possible. +

+ +
+ +

Examples

+ +

I uploaded a picture on Kakigoori, and the system attributed the ID 12ca0232-8515-4366-84d4-fa10a2e15069 to it!

+ +
+

https://kakigoori.dev/12ca0232-8515-4366-84d4-fa10a2e15069/auto
(Will display AVIF images first if available and compatible with your browser, or WebP images, or Jpegli images, or Jpeg images)

+ +

https://kakigoori.dev/12ca0232-8515-4366-84d4-fa10a2e15069/avif
(Will request the AVIF version of the image, or will return a 404 error if the version is not available. You can replace avif here with webp or jpegli)

+ +

https://kakigoori.dev/12ca0232-8515-4366-84d4-fa10a2e15069/original
(Will request the original image that was uploaded on Kakigoori)

+ +

https://kakigoori.dev/12ca0232-8515-4366-84d4-fa10a2e15069/height/300/auto
(Will request the image with an height of 300px, and will request the AVIF/WebP/Jpegli/Jpeg/PNG image depending on what is compatible with the browser and what's available)

+ +

https://kakigoori.dev/12ca0232-8515-4366-84d4-fa10a2e15069/width/300/auto
(Same thing, but with a width of 300px!)

+ +
+ +
+ +

Q&A

+ +
+ Who is using Kakigoori? + My websites (remilia.ch and ecdfeaa2.moe) all have their images stored on Kakigoori! I also use Kakigoori on some projects I worked on professionaly. +
+ +
+ Can I upload images to Kakigoori? + Well... currently, I'm kind of concerned about opening upload to anyone (I wouldn't want to have Backblaze sending me a nasty bill for using too much storage, or having them send me a gentle email to ask me why the fuck I'm hosting illicit content ^^).
+ So, my default answer is no, but if you're really interested to use Kakigoori, either self-host Kakigoori yourself, or if you want to use this hosted version and you're a friendly person, please send me an email to remilia@remilia.ch and I will see what I can do for you! +
+ +
+ How private is Kakigoori? + I'm not storing IPs, I'm not doing statistics, I'm not showing you any cookie banner because I'm not storing any, so on my side, I would say it's pretty compliant with privacy standards.
The only people that may know about you loading images from Kakigoori is Cloudflare, but at this point, with how much market share they have over the big websites of the Internet, they already know enough abut you ^^ +
+ +
+ Is Kakigoori open-source? + Yes! Please see the GitHub project for that! +
+ +
+ Who is behind Kakigoori? + I'm Remilia Da Costa Faro, your friendly nearby software developer and non-binary trans woman! +
+ +
+ ... why the name Kakigoori? + I was hungry when I created the project, and it was warm enough that I wanted some form of kakigoori.
+ Oh, and about the sunflowers?
+ They're pretty, I love them! (That's the only reason) +
+ +
+ How can I contact you if I want? + My email is always open (remilia@remilia.ch). Do note I may be a bit slow to reply, as replying to people scares me a bit ^^. +
+
+
+ + \ No newline at end of file