Initial commit!

This commit is contained in:
2024-09-10 11:23:04 +02:00
commit c40ee0a6bf
42 changed files with 1704 additions and 0 deletions

0
images/__init__.py Normal file
View File

3
images/admin.py Normal file
View File

@@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

6
images/apps.py Normal file
View File

@@ -0,0 +1,6 @@
from django.apps import AppConfig
class ImagesConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "images"

56
images/decorators.py Normal file
View File

@@ -0,0 +1,56 @@
from functools import wraps
from typing import Optional
from django.http import JsonResponse, HttpResponseForbidden
from images.models import Image, AuthorizationKey
def get_image(func):
@wraps(func)
def wrapper(request, *args, **kwargs):
image_id = kwargs["image_id"]
del kwargs["image_id"]
image = Image.objects.filter(id=image_id).first()
if image is None:
return JsonResponse({"error": "Image not found"}, status=404)
return func(request=request, image=image, *args, **kwargs)
return wrapper
def can_upload_image(func):
@wraps(func)
def wrapper(request, *args, **kwargs):
if "Authorization" not in request.headers:
return HttpResponseForbidden()
authorization_key: Optional[AuthorizationKey] = AuthorizationKey.objects.filter(
id=request.headers["Authorization"], can_upload_image=True
).first()
if not authorization_key:
return HttpResponseForbidden()
return func(request, *args, **kwargs)
return wrapper
def can_upload_variant(func):
@wraps(func)
def wrapper(request, *args, **kwargs):
if "Authorization" not in request.headers:
return HttpResponseForbidden()
authorization_key: Optional[AuthorizationKey] = AuthorizationKey.objects.filter(
id=request.headers["Authorization"], can_upload_variant=True
).first()
if not authorization_key:
return HttpResponseForbidden()
return func(request, *args, **kwargs)
return wrapper

View File

View File

View File

@@ -0,0 +1,35 @@
from django.core.management.base import BaseCommand
from images.models import Image, ImageVariant, ImageVariantTask
class Command(BaseCommand):
def handle(self, *args, **options):
images = Image.objects.filter(model_version=2).all()
images_len = len(images)
for index, image in enumerate(images):
print(f"Image {index}/{images_len}")
variants = ImageVariant.objects.filter(image=image).all()
variant_sizes = list(set(map(lambda x: (x.width, x.height), variants)))
for image_type in ["avif", "webp", "jpegli"]:
for variant_size in variant_sizes:
avif_variant = [
x
for x in variants
if x.width == variant_size[0]
and x.height == variant_size[1]
and x.file_type == image_type
]
if not avif_variant:
ImageVariantTask(
image=image,
height=variant_size[1],
width=variant_size[0],
original_file_type=image.imagevariant_set.filter(
is_full_size=True, file_type__in=["jpg", "png"]
)
.first()
.file_type,
file_type=image_type,
).save()

View File

@@ -0,0 +1,215 @@
import sys
from io import BytesIO
from PIL import Image as PILImage
import boto3
from botocore.config import Config
from django.core.management.base import BaseCommand, CommandError
from images.models import Image, ImageVariant
class Command(BaseCommand):
def get_b2_resource(self, endpoint, key_id, application_key):
b2 = boto3.resource(
service_name="s3",
endpoint_url=endpoint,
aws_access_key_id=key_id,
aws_secret_access_key=application_key,
config=Config(signature_version="s3v4"),
)
bucket = b2.Bucket("kakigoori")
return bucket
def handle(self, *args, **options):
images = Image.objects.filter(model_version=1).order_by("-creation_date").all()
images_len = len(images)
progress = 1
print(f"{images_len} images found")
for image in images:
print(f"Image {progress}/{images_len}")
try:
print("Upgrading image %s" % image.id)
bucket = self.get_b2_resource(
"https://s3.eu-central-003.backblazeb2.com",
"0032dcec6092f3e0000000021",
"K0039Xb1GE/An9P1ccK2B+19pXrKnnU",
)
print("Getting original image...")
original_image = BytesIO()
bucket.download_fileobj(
f"{image.backblaze_filepath}/original.jpg", original_image
)
original_image.seek(0)
with PILImage.open(original_image) as im:
image.height = im.height
image.width = im.width
image.save()
print("Copying original images...")
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/original.jpg",
},
f"{image.backblaze_filepath}/{image.width}-{image.height}/image.jpg",
)
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/thumbnail.jpg",
},
f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/image.jpg",
)
delete_objects_list = [f"{image.backblaze_filepath}/thumbnail.jpg"]
ImageVariant(
image=image,
height=image.height,
width=image.width,
is_full_size=True,
file_type="jpg",
).save()
ImageVariant(
image=image,
height=image.thumbnail_size[1],
width=image.thumbnail_size[0],
is_full_size=False,
file_type="jpg",
).save()
if image.is_jpegli_available:
print("Copying JPEGLI images...")
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/jpegli.jpg",
},
f"{image.backblaze_filepath}/{image.width}-{image.height}/jpegli.jpg",
)
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/thumbnail_jpegli.jpg",
},
f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/jpegli.jpg",
)
ImageVariant(
image=image,
height=image.height,
width=image.width,
is_full_size=True,
file_type="jpegli",
).save()
ImageVariant(
image=image,
height=image.thumbnail_size[0],
width=image.thumbnail_size[1],
is_full_size=False,
file_type="jpegli",
).save()
delete_objects_list += [
f"{image.backblaze_filepath}/jpegli.jpg",
f"{image.backblaze_filepath}/thumbnail_jpegli.jpg",
]
if image.is_avif_available:
print("Copying AVIF images...")
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/optimized.avif",
},
f"{image.backblaze_filepath}/{image.width}-{image.height}/image.avif",
)
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/thumbnail.avif",
},
f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/image.avif",
)
ImageVariant(
image=image,
height=image.height,
width=image.width,
is_full_size=True,
file_type="avif",
).save()
ImageVariant(
image=image,
height=image.thumbnail_size[0],
width=image.thumbnail_size[1],
is_full_size=False,
file_type="avif",
).save()
delete_objects_list += [
f"{image.backblaze_filepath}/optimized.avif",
f"{image.backblaze_filepath}/thumbnail.avif",
]
if image.is_webp_available:
print("Copying WebP images...")
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/optimized.webp",
},
f"{image.backblaze_filepath}/{image.width}-{image.height}/image.webp",
)
bucket.copy(
{
"Bucket": "kakigoori",
"Key": f"{image.backblaze_filepath}/thumbnail.webp",
},
f"{image.backblaze_filepath}/{image.thumbnail_size[0]}-{image.thumbnail_size[1]}/image.webp",
)
ImageVariant(
image=image,
height=image.height,
width=image.width,
is_full_size=True,
file_type="webp",
).save()
ImageVariant(
image=image,
height=image.thumbnail_size[0],
width=image.thumbnail_size[1],
is_full_size=False,
file_type="webp",
).save()
delete_objects_list += [
f"{image.backblaze_filepath}/optimized.webp",
f"{image.backblaze_filepath}/thumbnail.webp",
]
print("Saving...")
image.model_version = 2
image.save()
print("Deleting old files...")
bucket.delete_objects(
Delete={"Objects": [{"Key": x} for x in delete_objects_list]}
)
progress += 1
except KeyboardInterrupt:
sys.exit(1)
except:
progress += 1
pass

View File

@@ -0,0 +1,39 @@
# Generated by Django 5.0 on 2023-12-08 09:24
import django.utils.timezone
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Image",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"creation_date",
models.DateTimeField(default=django.utils.timezone.now),
),
("uploaded", models.BooleanField(default=False)),
("original_name", models.CharField(max_length=150)),
("original_mime_type", models.CharField(max_length=10)),
("original_md5", models.CharField(max_length=32)),
("is_mozjpeg_available", models.BooleanField(default=False)),
("is_webp_available", models.BooleanField(default=False)),
("is_avif_available", models.BooleanField(default=False)),
],
),
]

View File

@@ -0,0 +1,22 @@
# Generated by Django 5.0 on 2024-04-08 09:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('images', '0001_initial'),
]
operations = [
migrations.RemoveField(
model_name='image',
name='is_mozjpeg_available',
),
migrations.AddField(
model_name='image',
name='is_jpegli_available',
field=models.BooleanField(default=False),
),
]

View File

@@ -0,0 +1,50 @@
# Generated by Django 5.0 on 2024-06-22 13:56
import django.db.models.deletion
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('images', '0002_remove_image_is_mozjpeg_available_and_more'),
]
operations = [
migrations.CreateModel(
name='AuthorizationKeys',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.CharField(max_length=150)),
('can_upload', models.BooleanField(default=False)),
('can_convert', models.BooleanField(default=False)),
],
),
migrations.AddField(
model_name='image',
name='height',
field=models.IntegerField(default=0),
),
migrations.AddField(
model_name='image',
name='model_version',
field=models.IntegerField(default=1),
),
migrations.AddField(
model_name='image',
name='width',
field=models.IntegerField(default=0),
),
migrations.CreateModel(
name='ImageVariant',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('height', models.IntegerField()),
('width', models.IntegerField()),
('is_full_size', models.BooleanField(default=False)),
('file_type', models.CharField(max_length=10)),
('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='images.image')),
],
),
]

View File

@@ -0,0 +1,22 @@
# Generated by Django 5.0 on 2024-06-25 09:50
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("images", "0003_authorizationkeys_image_height_image_model_version_and_more"),
]
operations = [
migrations.RenameModel(
old_name="AuthorizationKeys",
new_name="AuthorizationKey",
),
migrations.RenameField(
model_name="authorizationkey",
old_name="can_convert",
new_name="can_upload_variant",
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 5.0 on 2024-06-25 09:50
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("images", "0004_rename_authorizationkeys_authorizationkey_and_more"),
]
operations = [
migrations.RenameField(
model_name="authorizationkey",
old_name="can_upload",
new_name="can_upload_image",
),
]

View File

@@ -0,0 +1,40 @@
# Generated by Django 5.0 on 2024-06-26 21:34
import django.db.models.deletion
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("images", "0005_rename_can_upload_authorizationkey_can_upload_image"),
]
operations = [
migrations.CreateModel(
name="ImageVariantTask",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("height", models.IntegerField()),
("width", models.IntegerField()),
("original_file_type", models.CharField(max_length=10)),
("file_type", models.CharField(max_length=10)),
(
"image",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="images.image"
),
),
],
),
]

View File

138
images/models.py Normal file
View File

@@ -0,0 +1,138 @@
import uuid
from io import BytesIO
from django.db import models
from django.utils import timezone
from images.utils import get_b2_resource
from PIL import Image as PILImage, ImageOps
class Image(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
creation_date = models.DateTimeField(default=timezone.now)
uploaded = models.BooleanField(default=False)
original_name = models.CharField(max_length=150)
original_mime_type = models.CharField(max_length=10)
original_md5 = models.CharField(max_length=32)
is_webp_available = models.BooleanField(default=False)
is_avif_available = models.BooleanField(default=False)
is_jpegli_available = models.BooleanField(default=False)
model_version = models.IntegerField(default=1)
height = models.IntegerField(default=0)
width = models.IntegerField(default=0)
@property
def thumbnail_size(self):
if self.height > self.width:
return int(600 * self.width / self.height), 600
else:
return 600, int(600 * self.height / self.width)
@property
def backblaze_filepath(self):
return f"{self.id.hex[:2]}/{self.id.hex[2:4]}/{self.id.hex}"
def create_variant_tasks(self, width, height, original_file_type):
ImageVariantTask(
image=self,
height=height,
width=width,
original_file_type=original_file_type,
file_type="avif",
).save()
ImageVariantTask(
image=self,
height=height,
width=width,
original_file_type=original_file_type,
file_type="webp",
).save()
ImageVariantTask(
image=self,
height=height,
width=width,
original_file_type=original_file_type,
file_type="jpegli",
).save()
def create_variant(self, width, height):
bucket = get_b2_resource()
original_image = BytesIO()
original_variant = self.imagevariant_set.filter(
is_full_size=True, file_type__in=["jpg", "png"]
).first()
bucket.download_fileobj(
f"{self.backblaze_filepath}/{original_variant.width}-{original_variant.height}/image.{original_variant.file_type}",
original_image,
)
original_image.seek(0)
resized_image = BytesIO()
file_extension = "jpg"
with PILImage.open(original_image) as im:
ImageOps.exif_transpose(im, in_place=True)
im.thumbnail((width, height))
if im.has_transparency_data:
try:
im.save(resized_image, "PNG")
file_extension = "png"
except OSError:
im.convert("RGB").save(resized_image, "JPEG")
else:
try:
im.save(resized_image, "JPEG")
except OSError:
im.convert("RGB").save(resized_image, "JPEG")
resized_image.seek(0)
bucket.upload_fileobj(
resized_image,
f"{self.backblaze_filepath}/{width}-{height}/image.{file_extension}",
)
image_variant = ImageVariant(
image=self,
height=height,
width=width,
is_full_size=False,
file_type=file_extension,
)
image_variant.save()
self.create_variant_tasks(width, height, file_extension)
return image_variant
class ImageVariant(models.Model):
image = models.ForeignKey(Image, on_delete=models.CASCADE)
height = models.IntegerField()
width = models.IntegerField()
is_full_size = models.BooleanField(default=False)
file_type = models.CharField(max_length=10)
class ImageVariantTask(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
created_at = models.DateTimeField(auto_now_add=True)
image = models.ForeignKey(Image, on_delete=models.CASCADE)
height = models.IntegerField()
width = models.IntegerField()
original_file_type = models.CharField(max_length=10)
file_type = models.CharField(max_length=10)
class AuthorizationKey(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
name = models.CharField(max_length=150)
can_upload_image = models.BooleanField(default=False)
can_upload_variant = models.BooleanField(default=False)

3
images/tests.py Normal file
View File

@@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

34
images/urls.py Normal file
View File

@@ -0,0 +1,34 @@
from django.urls import path
from . import views
urlpatterns = [
path("", views.index, name="index"),
path("upload", views.upload, name="images.upload"),
path(
"conversion_tasks/upload_variant",
views.upload_variant,
name="images.upload_variant",
),
path(
"conversion_tasks/<image_type>",
views.image_type_optimization_needed,
name="images.avif_optimization_needed",
),
path("<uuid:image_id>/<str:image_type>", views.get, name="images.get"),
path(
"<uuid:image_id>/<str:image_type>/thumbnail",
views.get_thumbnail,
name="images.get_thumbnail",
),
path(
"<uuid:image_id>/height/<int:height>/<str:image_type>",
views.get_image_with_height,
name="images.get_image_with_height",
),
path(
"<uuid:image_id>/width/<int:width>/<str:image_type>",
views.get_image_with_width,
name="images.get_image_with_width",
),
]

17
images/utils.py Normal file
View File

@@ -0,0 +1,17 @@
import boto3
from botocore.config import Config
from django.conf import settings
def get_b2_resource():
b2 = boto3.resource(
service_name="s3",
endpoint_url=settings.S3_ENDPOINT,
aws_access_key_id=settings.S3_KEY_ID,
aws_secret_access_key=settings.S3_ACCESS_KEY,
config=Config(signature_version="s3v4"),
)
bucket = b2.Bucket(settings.S3_BUCKET)
return bucket

241
images/views.py Normal file
View File

@@ -0,0 +1,241 @@
import hashlib
from PIL import Image as PILImage
from PIL import JpegImagePlugin
from django.conf import settings
from django.http import (
JsonResponse,
HttpResponseBadRequest,
HttpResponseNotFound,
)
from django.shortcuts import redirect, render
from django.views.decorators.csrf import csrf_exempt
from images.decorators import (
get_image,
can_upload_variant,
can_upload_image,
)
from images.models import Image, ImageVariant, ImageVariantTask
from images.utils import get_b2_resource
JpegImagePlugin._getmp = lambda x: None
def index(request):
return render(request, "index.html")
@csrf_exempt
@can_upload_image
def upload(request):
file = request.FILES["file"]
bucket = get_b2_resource()
file_md5_hash = hashlib.file_digest(file, "md5").hexdigest()
file.seek(0)
with PILImage.open(file) as im:
height, width = (im.height, im.width)
if im.format == "JPEG":
file_extension = "jpg"
elif im.format == "PNG":
file_extension = "png"
else:
return {"created": False, "error": "Uploaded file should be JPEG or PNG"}
file.seek(0)
same_md5_image = Image.objects.filter(original_md5=file_md5_hash).first()
if same_md5_image:
return {"created": False, "id": same_md5_image.id}
image = Image(
original_name=file.name,
original_mime_type=file.content_type,
original_md5=file_md5_hash,
height=height,
width=width,
model_version=2,
)
image.save()
bucket.upload_fileobj(
file, f"{image.backblaze_filepath}/{width}-{height}/image.{file_extension}"
)
ImageVariant.objects.get_or_create(
image=image,
height=height,
width=width,
file_type=file_extension,
is_full_size=True,
)
image.create_variant_tasks(image.width, image.height, file_extension)
image.uploaded = True
image.save()
return JsonResponse({"created": True, "id": image.id})
@can_upload_variant
def image_type_optimization_needed(request, image_type):
tasks = ImageVariantTask.objects.filter(file_type=image_type).all()
return JsonResponse(
{
"variants": [
{
"image_id": task.image.id,
"task_id": task.id,
"height": task.height,
"width": task.width,
"file_type": task.original_file_type,
}
for task in tasks
]
}
)
@csrf_exempt
@can_upload_variant
def upload_variant(request):
if "task_id" not in request.POST:
return HttpResponseBadRequest()
task = ImageVariantTask.objects.filter(id=request.POST["task_id"]).first()
if not task:
return HttpResponseNotFound()
image = task.image
height = task.height
width = task.width
file_type = task.file_type
file = request.FILES["file"]
if file_type == "jpegli":
file_name = "jpegli.jpg"
else:
file_name = "image." + file_type
upload_path = f"{image.backblaze_filepath}/{width}-{height}/{file_name}"
bucket = get_b2_resource()
bucket.upload_fileobj(file, upload_path)
ImageVariant.objects.get_or_create(
image=image,
height=height,
width=width,
file_type=file_type,
is_full_size=(height == image.height and width == image.width),
)
task.delete()
return JsonResponse({"status": "ok"})
def image_with_size(request, image, width, height, image_type):
image_variants = ImageVariant.objects.filter(
image=image, height=height, width=width
)
if image_type != "auto":
if image_type == "original":
image_variants = image_variants.filter(file_type__in=["jpg", "png"])
else:
image_variants = image_variants.filter(file_type=image_type)
variants = image_variants.all()
if not variants:
if image_type != "auto" and image_type != "original":
return JsonResponse({"error": "Image version not available"}, status=404)
else:
image_variant = image.create_variant(width, height)
return redirect(
f"{settings.S3_PUBLIC_BASE_PATH}/{image.backblaze_filepath}/{width}-{height}/image.{image_variant.file_type}"
)
if image_type == "auto":
variants_preferred_order = ["avif", "webp", "jpegli", "jpg", "png"]
elif image_type == "original":
variants_preferred_order = ["jpg", "png"]
else:
variants_preferred_order = [image_type]
accept_header = request.headers.get("Accept", default="")
for file_type in variants_preferred_order:
if (
file_type == "avif"
and image_type == "auto"
and "image/avif" not in accept_header
):
continue
if (
file_type == "webp"
and image_type == "auto"
and "image/webp" not in accept_header
):
continue
variant = [x for x in image_variants if x.file_type == file_type]
if not variant:
continue
variant = variant[0]
if file_type == "jpegli":
file_name = "jpegli.jpg"
else:
file_name = "image." + file_type
return redirect(
f"{settings.S3_PUBLIC_BASE_PATH}/{image.backblaze_filepath}/{variant.width}-{variant.height}/{file_name}"
)
return HttpResponseNotFound()
@get_image
def get_image_with_height(request, image, height, image_type):
if height >= image.height:
height = image.height
width = image.width
else:
width = int(height * image.width / image.height)
return image_with_size(request, image, width, height, image_type)
@get_image
def get_image_with_width(request, image, width, image_type):
if width >= image.width:
width = image.width
height = image.height
else:
height = int(width * image.height / image.width)
return image_with_size(request, image, width, height, image_type)
@get_image
def get(request, image, image_type):
return image_with_size(request, image, image.width, image.height, image_type)
@get_image
def get_thumbnail(request, image, image_type):
width, height = image.thumbnail_size
return image_with_size(request, image, width, height, image_type)