feat(recognition): add local face/animal detection pipeline
Add a face/animal recognition feature: local ONNX detection (YuNet+SFace for faces with automatic person clustering, YOLOX-Nano for animal species) behind an IRecognitionService seam mirroring IPlaceSearchService's local/ remote proxy pattern, with a separate engine seam so the ONNX backend can be swapped later. Wired into the import/sync pipeline via TaskType.IMAGE_ANALYSIS with a visible progress task, a startup backfill for existing libraries, and a cheap perceptual-hash skip so metadata-only edits don't retrigger detection. Detected regions are stored (new person/media_face_region tables) and shown read-only in MediaInfoPane. Bundled ONNX weights (MIT/Apache-2.0) are tracked via Git LFS. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JzvA5ySQUsYrMUTj7sHxFA
This commit is contained in:
@@ -19,3 +19,7 @@
|
||||
*.jpeg binary
|
||||
*.ico binary
|
||||
*.icns binary
|
||||
|
||||
# Bundled ONNX recognition models (src/main/resources/models/recognition/) — tens of MB each, stored via
|
||||
# Git LFS rather than directly in history so a plain clone stays small unless LFS objects are pulled.
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
## Shell
|
||||
|
||||
* [ ] nouveau menu pour help / credit
|
||||
* [ ] prevoir des scenario pour les aides guide
|
||||
* [ ] prevoir des scenarios pour les aides guide
|
||||
|
||||
## Status bar
|
||||
|
||||
@@ -33,11 +33,15 @@
|
||||
|
||||
## Photos detail
|
||||
|
||||
* [ ] implementer le panel info en slide * [ ] Info: ajouter les infos en lecteur * [ ] Info: editer date * [ ] Info: editer lieux
|
||||
* [ ] afficher star + modifier star
|
||||
* [ ] zoom in / out sur la photo : fonctionne avec la souris, mais pas terrible avec le trackpad
|
||||
* [x] tenir compte de l'orientation
|
||||
* [ ] Animation de L'image qd le panel info arrive
|
||||
* [x] implementer le panel info en slide
|
||||
* [x] Info: ajouter les infos en lecteur
|
||||
* [x] Info: editer date
|
||||
* [x] Info: editer lieux
|
||||
* [x] la carte scintille a chaque changement de photos.
|
||||
* [x] afficher star + modifier star
|
||||
* [x] tenir compte de l'orientation
|
||||
* [x] cache image dans PhotoDetailsPane
|
||||
|
||||
## Notifications
|
||||
@@ -60,6 +64,6 @@
|
||||
* [x] ecrire le gps
|
||||
* [ ] ecrire les tags
|
||||
* [x] ecrire la date
|
||||
* [ ] ecrire rating
|
||||
* [x] ecrire rating
|
||||
* [ ] definir un writer pour tous les fichier pas supporte avec exif tool: binaire inclus dans l'app et installation au premier demarrage. notif en erreur si le
|
||||
binaire n'est plus trouve.
|
||||
|
||||
@@ -35,6 +35,16 @@
|
||||
<byte-buddy.version>1.18.12</byte-buddy.version>
|
||||
<commons-codec.version>1.22.1</commons-codec.version>
|
||||
<commons-imaging.version>1.0.0-alpha6</commons-imaging.version>
|
||||
<!--
|
||||
Local face/animal recognition (YuNet detection, SFace embedding, YOLOX-Nano animal
|
||||
detection) runs on Microsoft's own ONNX Runtime Java API directly, not through Deep
|
||||
Java Library: every model here needs fully custom pre/post-processing (letterboxing,
|
||||
anchor decoding, NMS, 5-point face alignment), so DJL's Translator/ZooModel machinery
|
||||
would add an abstraction layer with nothing left for it to actually do. This jar bundles
|
||||
native libraries for every major desktop platform in one artifact — no per-OS classifier,
|
||||
no runtime download.
|
||||
-->
|
||||
<onnxruntime.version>1.21.1</onnxruntime.version>
|
||||
<!--
|
||||
Scene-graph inspector, wired in by DevTools and dormant unless pholio.dev-tools.enabled is
|
||||
set. Built for Java 21 / JavaFX 23; Pholio runs it on 26 in classpath mode, so it relies on
|
||||
@@ -256,6 +266,13 @@
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>${commons-codec.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- ==================== Local ML (recognition) ==================== -->
|
||||
<dependency>
|
||||
<groupId>com.microsoft.onnxruntime</groupId>
|
||||
<artifactId>onnxruntime</artifactId>
|
||||
<version>${onnxruntime.version}</version>
|
||||
</dependency>
|
||||
<!-- =========================== Caching =========================== -->
|
||||
<dependency>
|
||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
||||
|
||||
@@ -8,7 +8,8 @@ package org.icroco.pholio.domain.library;
|
||||
public enum EMediaFileProcessingFlag {
|
||||
METADATA_GENERATED(0),
|
||||
THUMBNAIL_GENERATED(1),
|
||||
GEO_REFERENCE_GENERATED(2);
|
||||
GEO_REFERENCE_GENERATED(2),
|
||||
FACE_DETECTED(3);
|
||||
|
||||
private final int bit;
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package org.icroco.pholio.domain.recognition;
|
||||
|
||||
/**
|
||||
* A detected region's location within its image, in the MWG-RS {@code stArea} convention: {@code x}/{@code y}
|
||||
* is the box's <b>center</b> point, {@code w}/{@code h} its size — all four fractions of the full image's own
|
||||
* width/height (0..1), not pixels. Chosen deliberately so {@code media_face_region}'s stored columns are
|
||||
* already in the exact shape an {@code mwg-rs:Area} XMP struct needs on write, with no conversion either way.
|
||||
*/
|
||||
public record BoundingBox(double x, double y, double w, double h) {
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package org.icroco.pholio.domain.recognition;
|
||||
|
||||
import org.jilt.Builder;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
|
||||
/**
|
||||
* One face or animal found in a photo by an {@code IRecognitionService} call — not yet persisted, and not yet
|
||||
* linked to any {@link Person}; see {@link MediaFaceRegion} for the persisted, DB-identified counterpart.
|
||||
*
|
||||
* @param embedding {@link EEntityKind#PERSON} only — the face embedding vector clustering compares
|
||||
* across photos to group the same identity; {@code null} for {@link EEntityKind#ANIMAL}
|
||||
* @param label {@link EEntityKind#ANIMAL} only — the detected species (e.g. {@code "dog"}); {@code
|
||||
* null} for {@link EEntityKind#PERSON}, whose identity is a {@link Person}, not a label
|
||||
* @param sourceProvider which {@code IRecognitionService}/engine produced this — {@code "local-djl"}, or a
|
||||
* configured remote provider's own name
|
||||
*/
|
||||
@Builder(factoryMethod = "detectedRegion")
|
||||
public record DetectedRegion(EEntityKind kind,
|
||||
BoundingBox box,
|
||||
double confidence,
|
||||
float @Nullable [] embedding,
|
||||
@Nullable String label,
|
||||
String sourceProvider) {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package org.icroco.pholio.domain.recognition;
|
||||
|
||||
/** What a {@link DetectedRegion}/{@link MediaFaceRegion} identifies — a human face or an animal. */
|
||||
public enum EEntityKind {
|
||||
PERSON,
|
||||
ANIMAL
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package org.icroco.pholio.domain.recognition;
|
||||
|
||||
import org.jilt.Builder;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* A {@link DetectedRegion} once persisted against a {@code MediaFile} — see {@link DetectedRegion} for what
|
||||
* each field means; the two differ only in that this one carries a database identity and an optional link to
|
||||
* the {@link Person} {@code FaceClusteringService} has grouped it under.
|
||||
*
|
||||
* @param personId {@code null} until {@code FaceClusteringService} links this region to a {@link Person}
|
||||
* (new or existing) — {@link EEntityKind#ANIMAL} rows never get one in this iteration, since
|
||||
* individual animal identity isn't attempted yet, only the species {@link #label}
|
||||
* @param confirmed set once the future person-management panel confirms this region — never set by automatic
|
||||
* detection/clustering; see {@code MediaMetadataEditService.updateFaceRegions}, which is the
|
||||
* only path that ever writes this region out to the file's own XMP packet
|
||||
*/
|
||||
@Builder(factoryMethod = "mediaFaceRegion", toBuilder = "from")
|
||||
public record MediaFaceRegion(@Nullable Long id,
|
||||
Long mediaFileId,
|
||||
@Nullable Long personId,
|
||||
EEntityKind kind,
|
||||
BoundingBox area,
|
||||
double confidence,
|
||||
float @Nullable [] embedding,
|
||||
@Nullable String label,
|
||||
String sourceProvider,
|
||||
boolean confirmed,
|
||||
Instant detectedAt) {
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package org.icroco.pholio.domain.recognition;
|
||||
|
||||
import org.jilt.Builder;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* One recognized identity — a specific person or a specific animal — shared across every
|
||||
* {@link MediaFaceRegion} that {@code FaceClusteringService} has linked to it.
|
||||
*
|
||||
* @param id {@code null} before the row is persisted
|
||||
* @param name {@code null} until named — either by {@code FaceClusteringService} minting a fresh, unnamed
|
||||
* cluster, or before the future person-management panel lets a user confirm one
|
||||
*/
|
||||
@Builder(factoryMethod = "person", toBuilder = "from")
|
||||
public record Person(@Nullable Long id,
|
||||
EEntityKind kind,
|
||||
@Nullable String name,
|
||||
Instant createdAt) {
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package org.icroco.pholio.domain.recognition;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** What one {@code IRecognitionService.analyze} call found in a single image. */
|
||||
public record RecognitionResult(List<DetectedRegion> regions) {
|
||||
|
||||
public static RecognitionResult empty() {
|
||||
return new RecognitionResult(List.of());
|
||||
}
|
||||
}
|
||||
@@ -161,7 +161,9 @@ public class MediaAnalysisService {
|
||||
thumbnailGenerator.decode(target, ImageFormat.JPEG).ifPresentOrElse(
|
||||
cached -> taskService.execute(TaskType.IMAGE_ANALYSIS, () -> hashAndPublish(mediaFileId, cached, thumbnailBatch)),
|
||||
() -> {
|
||||
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId));
|
||||
// Never assume "unchanged" when the signal that would tell us (a freshly computed
|
||||
// phash) was never even attempted — see MediaFileAnalyzedEvent's own javadoc.
|
||||
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId, false));
|
||||
thumbnailBatch.completedOne();
|
||||
});
|
||||
return;
|
||||
@@ -170,7 +172,7 @@ public class MediaAnalysisService {
|
||||
Optional<BufferedImage> decoded = thumbnailGenerator.decode(absolute, format.get());
|
||||
if (decoded.isEmpty()) {
|
||||
log.debug("No pixels obtainable for '{}'; no thumbnail generated", absolute);
|
||||
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId));
|
||||
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId, false));
|
||||
thumbnailBatch.completedOne();
|
||||
return;
|
||||
}
|
||||
@@ -196,15 +198,23 @@ public class MediaAnalysisService {
|
||||
}
|
||||
}
|
||||
|
||||
/** Hamming distance at or below which two perceptual hashes are treated as "the same picture". */
|
||||
private static final int UNCHANGED_PHASH_THRESHOLD = 2;
|
||||
|
||||
private void hashAndPublish(Long mediaFileId, BufferedImage thumbnail, TaskService.BatchTask thumbnailBatch) {
|
||||
try {
|
||||
long hash = perceptualHasher.phash(thumbnail);
|
||||
mediaFileRepository.findById(mediaFileId).ifPresent(entity -> {
|
||||
long hash = perceptualHasher.phash(thumbnail);
|
||||
boolean unchanged = mediaFileRepository.findById(mediaFileId).map(entity -> {
|
||||
Long previousHash = entity.getPhash();
|
||||
entity.setPhash(hash);
|
||||
entity.setProcessingFlags(MediaFile.withBit(entity.getProcessingFlags(), EMediaFileProcessingFlag.THUMBNAIL_GENERATED));
|
||||
mediaFileRepository.save(entity);
|
||||
});
|
||||
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId));
|
||||
// A metadata-only rewrite (rating, GPS, capture date) leaves pixels — and so the phash —
|
||||
// alone; only a genuine content change moves it. See MediaFileAnalyzedEvent's own javadoc
|
||||
// for why "unknown" (no previous hash) must never be read as "unchanged".
|
||||
return previousHash != null && PerceptualHasher.hammingDistance(previousHash, hash) <= UNCHANGED_PHASH_THRESHOLD;
|
||||
}).orElse(false);
|
||||
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId, unchanged));
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
log.warn("Perceptual hashing failed unexpectedly: {}", e.toString(), e);
|
||||
|
||||
@@ -5,6 +5,14 @@ package org.icroco.pholio.infra.library;
|
||||
* finishes for one {@link org.icroco.pholio.domain.library.MediaFile} — up to twice per file, since the
|
||||
* two run independently on their own pools. Listeners are expected to debounce: a large import fires this
|
||||
* for every file, and nothing needs a UI refresh on each one individually.
|
||||
*
|
||||
* @param pixelsLikelyUnchanged {@code true} when the newly computed perceptual hash is close enough to the
|
||||
* file's previous one that its pixels almost certainly did not change — e.g. a
|
||||
* metadata-only rewrite (rating, GPS, capture date) touched the file's bytes
|
||||
* without touching what it shows. {@code false} whenever that cannot be
|
||||
* determined (new file, no previous hash) or the pixels did change — never
|
||||
* assumed by default, so a listener that skips expensive re-work on this signal
|
||||
* (see {@code MediaFileRecognitionTrigger}) only ever does so when genuinely safe.
|
||||
*/
|
||||
public record MediaFileAnalyzedEvent(Long mediaFileId) {
|
||||
public record MediaFileAnalyzedEvent(Long mediaFileId, boolean pixelsLikelyUnchanged) {
|
||||
}
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package org.icroco.pholio.infra.persistence.recognition;
|
||||
|
||||
import lombok.*;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
import org.springframework.data.annotation.Id;
|
||||
import org.springframework.data.relational.core.mapping.Column;
|
||||
import org.springframework.data.relational.core.mapping.Table;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/** One detected face/animal bounding box against a {@code media_file} row — see {@link PersonEntity}. */
|
||||
@Table("media_face_region")
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class MediaFaceRegionEntity {
|
||||
|
||||
@Id
|
||||
@Column("id")
|
||||
private @Nullable Long id;
|
||||
|
||||
@Column("media_file_id")
|
||||
private Long mediaFileId;
|
||||
|
||||
@Column("person_id")
|
||||
private @Nullable Long personId;
|
||||
|
||||
@Column("kind")
|
||||
private String kind;
|
||||
|
||||
@Column("area_x")
|
||||
private double areaX;
|
||||
|
||||
@Column("area_y")
|
||||
private double areaY;
|
||||
|
||||
@Column("area_w")
|
||||
private double areaW;
|
||||
|
||||
@Column("area_h")
|
||||
private double areaH;
|
||||
|
||||
@Column("confidence")
|
||||
private double confidence;
|
||||
|
||||
@Column("embedding")
|
||||
private byte @Nullable [] embedding;
|
||||
|
||||
@Column("label")
|
||||
private @Nullable String label;
|
||||
|
||||
@Column("source_provider")
|
||||
private String sourceProvider;
|
||||
|
||||
@Column("confirmed")
|
||||
private boolean confirmed;
|
||||
|
||||
@Column("detected_at")
|
||||
private Instant detectedAt;
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package org.icroco.pholio.infra.persistence.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.BoundingBox;
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.domain.recognition.MediaFaceRegion;
|
||||
import org.icroco.pholio.infra.recognition.EmbeddingCodec;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.Mapping;
|
||||
|
||||
@Mapper(componentModel = "spring", imports = EmbeddingCodec.class)
|
||||
public interface MediaFaceRegionMapper {
|
||||
|
||||
@Mapping(target = "area", expression = "java(toBoundingBox(entity))")
|
||||
@Mapping(target = "embedding", expression = "java(EmbeddingCodec.fromBytes(entity.getEmbedding()))")
|
||||
MediaFaceRegion toDomain(MediaFaceRegionEntity entity);
|
||||
|
||||
@Mapping(target = "id", ignore = true)
|
||||
@Mapping(target = "areaX", source = "area.x")
|
||||
@Mapping(target = "areaY", source = "area.y")
|
||||
@Mapping(target = "areaW", source = "area.w")
|
||||
@Mapping(target = "areaH", source = "area.h")
|
||||
@Mapping(target = "embedding", expression = "java(EmbeddingCodec.toBytes(domain.embedding()))")
|
||||
MediaFaceRegionEntity toEntity(MediaFaceRegion domain);
|
||||
|
||||
default BoundingBox toBoundingBox(MediaFaceRegionEntity entity) {
|
||||
return new BoundingBox(entity.getAreaX(), entity.getAreaY(), entity.getAreaW(), entity.getAreaH());
|
||||
}
|
||||
|
||||
default String map(EEntityKind kind) {
|
||||
return kind.name();
|
||||
}
|
||||
|
||||
default EEntityKind map(String kind) {
|
||||
return EEntityKind.valueOf(kind);
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package org.icroco.pholio.infra.persistence.recognition;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.data.repository.ListCrudRepository;
|
||||
|
||||
public interface MediaFaceRegionRepository extends ListCrudRepository<MediaFaceRegionEntity, Long> {
|
||||
|
||||
List<MediaFaceRegionEntity> findByMediaFileId(Long mediaFileId);
|
||||
|
||||
List<MediaFaceRegionEntity> findByMediaFileIdIn(Collection<Long> mediaFileIds);
|
||||
|
||||
/** Clears a file's previous detections before a re-detection writes its new set — mirrors {@code MediaFileTagRepository}. */
|
||||
void deleteByMediaFileIdIn(Collection<Long> mediaFileIds);
|
||||
|
||||
/** Unclustered faces — what {@code FaceClusteringService} still needs to link to a {@code Person}. */
|
||||
List<MediaFaceRegionEntity> findByKindAndPersonIdIsNull(String kind);
|
||||
|
||||
List<MediaFaceRegionEntity> findByPersonId(Long personId);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package org.icroco.pholio.infra.persistence.recognition;
|
||||
|
||||
import lombok.*;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
import org.springframework.data.annotation.Id;
|
||||
import org.springframework.data.relational.core.mapping.Column;
|
||||
import org.springframework.data.relational.core.mapping.Table;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/** One recognized identity (a person or an animal) — see {@link MediaFaceRegionEntity} for its detections. */
|
||||
@Table("person")
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class PersonEntity {
|
||||
|
||||
@Id
|
||||
@Column("id")
|
||||
private @Nullable Long id;
|
||||
|
||||
@Column("kind")
|
||||
private String kind;
|
||||
|
||||
@Column("name")
|
||||
private @Nullable String name;
|
||||
|
||||
@Column("created_at")
|
||||
private Instant createdAt;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package org.icroco.pholio.infra.persistence.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.domain.recognition.Person;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.Mapping;
|
||||
|
||||
@Mapper(componentModel = "spring")
|
||||
public interface PersonMapper {
|
||||
|
||||
Person toDomain(PersonEntity entity);
|
||||
|
||||
@Mapping(target = "id", ignore = true)
|
||||
PersonEntity toEntity(Person domain);
|
||||
|
||||
default String map(EEntityKind kind) {
|
||||
return kind.name();
|
||||
}
|
||||
|
||||
default EEntityKind map(String kind) {
|
||||
return EEntityKind.valueOf(kind);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package org.icroco.pholio.infra.persistence.recognition;
|
||||
|
||||
import org.springframework.data.repository.ListCrudRepository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface PersonRepository extends ListCrudRepository<PersonEntity, Long> {
|
||||
|
||||
List<PersonEntity> findByKind(String kind);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
/**
|
||||
* Which wire contract a {@link RecognitionProviderConfig} speaks. Unlike {@code EGeocodingProviderKind},
|
||||
* there is no well-known third-party face/animal-recognition API to standardise on, so — for now — a remote
|
||||
* recognition provider is expected to be a small HTTP service speaking Pholio's own contract: {@code POST}
|
||||
* to {@link RecognitionProviderConfig#urlTemplate()} a JSON body {@code {"apiKey", "width", "height",
|
||||
* "imageBase64"}} (the decoded image, JPEG-encoded, then base64), and reply with a JSON array of
|
||||
* {@code {"kind":"PERSON"|"ANIMAL","x","y","w","h","confidence","label"}} (the same normalized {@code
|
||||
* stArea} center/size convention {@link org.icroco.pholio.domain.recognition.BoundingBox} uses).
|
||||
*
|
||||
* <p>Kept as a real enum rather than inlining {@link #HTTP_JSON} everywhere so a second, differently-shaped
|
||||
* remote contract can be added later as one more case, the same reason {@code EGeocodingProviderKind} does.
|
||||
*/
|
||||
public enum ERecognitionProviderKind {
|
||||
|
||||
HTTP_JSON
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.jspecify.annotations.Nullable;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
/**
|
||||
* {@code float[]} embedding vector <-> the little-endian {@code byte[]} stored in
|
||||
* {@code media_face_region.embedding} — H2/JDBC has no native vector/array column type, and a plain byte
|
||||
* buffer needs no third-party (de)serialization library for something this small (typically 128-512 floats).
|
||||
*/
|
||||
public final class EmbeddingCodec {
|
||||
|
||||
private EmbeddingCodec() {
|
||||
}
|
||||
|
||||
public static byte @Nullable [] toBytes(float @Nullable [] embedding) {
|
||||
if (embedding == null) {
|
||||
return null;
|
||||
}
|
||||
ByteBuffer buffer = ByteBuffer.allocate(embedding.length * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
|
||||
for (float value : embedding) {
|
||||
buffer.putFloat(value);
|
||||
}
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
public static float @Nullable [] fromBytes(byte @Nullable [] bytes) {
|
||||
if (bytes == null) {
|
||||
return null;
|
||||
}
|
||||
ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
|
||||
float[] embedding = new float[bytes.length / Float.BYTES];
|
||||
for (int i = 0; i < embedding.length; i++) {
|
||||
embedding[i] = buffer.getFloat();
|
||||
}
|
||||
return embedding;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionEntity;
|
||||
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionRepository;
|
||||
import org.icroco.pholio.infra.persistence.recognition.PersonEntity;
|
||||
import org.icroco.pholio.infra.persistence.recognition.PersonRepository;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Groups unnamed {@link EEntityKind#PERSON} face regions into {@link PersonEntity} identities by embedding
|
||||
* similarity, entirely automatically — no manual step, the "Person 1", "Person 2"... clusters the future
|
||||
* person-management panel will let a user name. Animal regions are never clustered: individual animal
|
||||
* identity isn't attempted in this iteration, only the species label already on the region.
|
||||
*
|
||||
* <p>Brute-force cosine similarity against one centroid per existing cluster, not a full pairwise
|
||||
* comparison against every previously-seen face — acceptable at "one photo library" scale (thousands, not
|
||||
* millions, of faces); see the implementation plan's own risk notes for when that would need revisiting.
|
||||
*/
|
||||
@Service
|
||||
public class FaceClusteringService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(FaceClusteringService.class);
|
||||
|
||||
/** SFace's own calibrated same-identity cosine threshold (OpenCV Zoo's face_recognition_sface README). */
|
||||
private static final double DEFAULT_THRESHOLD = 0.363;
|
||||
|
||||
private final MediaFaceRegionRepository regionRepository;
|
||||
private final PersonRepository personRepository;
|
||||
private final AppPreferences preferences;
|
||||
|
||||
public FaceClusteringService(MediaFaceRegionRepository regionRepository, PersonRepository personRepository,
|
||||
AppPreferences preferences) {
|
||||
this.regionRepository = regionRepository;
|
||||
this.personRepository = personRepository;
|
||||
this.preferences = preferences;
|
||||
}
|
||||
|
||||
/** Links every unlinked {@link EEntityKind#PERSON} region to an existing or freshly-minted {@link PersonEntity}. */
|
||||
public void clusterUnnamedPersons() {
|
||||
List<MediaFaceRegionEntity> unassigned = regionRepository.findByKindAndPersonIdIsNull(EEntityKind.PERSON.name());
|
||||
if (unassigned.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
double threshold = preferences.getValueOr("recognition", "person-cluster-threshold", Double.class, DEFAULT_THRESHOLD);
|
||||
|
||||
List<Cluster> clusters = loadExistingClusters();
|
||||
List<MediaFaceRegionEntity> toSave = new ArrayList<>();
|
||||
for (MediaFaceRegionEntity region : unassigned) {
|
||||
float[] embedding = EmbeddingCodec.fromBytes(region.getEmbedding());
|
||||
if (embedding == null) {
|
||||
log.warn("Person region {} has no embedding, cannot cluster it", region.getId());
|
||||
continue;
|
||||
}
|
||||
Cluster best = bestMatch(clusters, embedding, threshold);
|
||||
if (best == null) {
|
||||
best = new Cluster(createPerson(), embedding.clone(), 1);
|
||||
clusters.add(best);
|
||||
}
|
||||
else {
|
||||
best.accumulate(embedding);
|
||||
}
|
||||
region.setPersonId(best.personId());
|
||||
toSave.add(region);
|
||||
}
|
||||
if (!toSave.isEmpty()) {
|
||||
regionRepository.saveAll(toSave);
|
||||
log.info("Clustered {} face region(s) into {} person(s)", toSave.size(), clusters.size());
|
||||
}
|
||||
}
|
||||
|
||||
private List<Cluster> loadExistingClusters() {
|
||||
List<Cluster> clusters = new ArrayList<>();
|
||||
for (PersonEntity person : personRepository.findByKind(EEntityKind.PERSON.name())) {
|
||||
Long personId = Objects.requireNonNull(person.getId(), "A persisted PersonEntity always has an id");
|
||||
List<float[]> embeddings = regionRepository.findByPersonId(personId).stream()
|
||||
.map(region -> EmbeddingCodec.fromBytes(region.getEmbedding()))
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
if (!embeddings.isEmpty()) {
|
||||
clusters.add(new Cluster(personId, mean(embeddings), embeddings.size()));
|
||||
}
|
||||
}
|
||||
return clusters;
|
||||
}
|
||||
|
||||
private static @Nullable Cluster bestMatch(List<Cluster> clusters, float[] embedding, double threshold) {
|
||||
Cluster best = null;
|
||||
double bestScore = threshold;
|
||||
for (Cluster cluster : clusters) {
|
||||
double score = cosineSimilarity(cluster.centroid(), embedding);
|
||||
if (score >= bestScore) {
|
||||
best = cluster;
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
private Long createPerson() {
|
||||
PersonEntity saved = personRepository.save(PersonEntity.builder()
|
||||
.kind(EEntityKind.PERSON.name())
|
||||
.name(null)
|
||||
.createdAt(Instant.now())
|
||||
.build());
|
||||
return Objects.requireNonNull(saved.getId(), "A freshly saved PersonEntity always has an id");
|
||||
}
|
||||
|
||||
private static float[] mean(List<float[]> embeddings) {
|
||||
float[] mean = new float[embeddings.getFirst().length];
|
||||
for (float[] embedding : embeddings) {
|
||||
for (int i = 0; i < mean.length; i++) {
|
||||
mean[i] += embedding[i];
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < mean.length; i++) {
|
||||
mean[i] /= embeddings.size();
|
||||
}
|
||||
return mean;
|
||||
}
|
||||
|
||||
private static double cosineSimilarity(float[] a, float[] b) {
|
||||
double dot = 0, normA = 0, normB = 0;
|
||||
for (int i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
if (normA == 0 || normB == 0) {
|
||||
return 0;
|
||||
}
|
||||
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||
}
|
||||
|
||||
/** A running centroid — its own field, mutated in place as more regions join it within one clustering pass. */
|
||||
private static final class Cluster {
|
||||
private final Long personId;
|
||||
private final float[] centroid;
|
||||
private int count;
|
||||
|
||||
private Cluster(Long personId, float[] centroid, int count) {
|
||||
this.personId = personId;
|
||||
this.centroid = centroid;
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
private Long personId() {
|
||||
return personId;
|
||||
}
|
||||
|
||||
private float[] centroid() {
|
||||
return centroid;
|
||||
}
|
||||
|
||||
private void accumulate(float[] embedding) {
|
||||
count++;
|
||||
for (int i = 0; i < centroid.length; i++) {
|
||||
centroid[i] += (embedding[i] - centroid[i]) / count;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.domain.recognition.MediaFaceRegion;
|
||||
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionEntity;
|
||||
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionMapper;
|
||||
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionRepository;
|
||||
import org.icroco.pholio.infra.persistence.recognition.PersonEntity;
|
||||
import org.icroco.pholio.infra.persistence.recognition.PersonRepository;
|
||||
import org.jspecify.annotations.Nullable;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Reads and replaces a {@code MediaFile}'s detected regions — used by {@link MediaRecognitionService} (writes,
|
||||
* after each detection pass), {@code MediaInfoPane} (reads, for its persons/animals row) and the future
|
||||
* person-management panel.
|
||||
*/
|
||||
@Service
|
||||
public class FaceRegionQueryService {
|
||||
|
||||
private final MediaFaceRegionRepository repository;
|
||||
private final MediaFaceRegionMapper mapper;
|
||||
private final PersonRepository personRepository;
|
||||
|
||||
public FaceRegionQueryService(MediaFaceRegionRepository repository, MediaFaceRegionMapper mapper,
|
||||
PersonRepository personRepository) {
|
||||
this.repository = repository;
|
||||
this.mapper = mapper;
|
||||
this.personRepository = personRepository;
|
||||
}
|
||||
|
||||
public List<MediaFaceRegion> findByMediaFile(Long mediaFileId) {
|
||||
return repository.findByMediaFileId(mediaFileId).stream().map(mapper::toDomain).toList();
|
||||
}
|
||||
|
||||
/** One entry per detected region, name resolved for {@link EEntityKind#PERSON} rows already linked to a named {@code Person}. */
|
||||
public List<DisplayRegion> findDisplayRegionsFor(Long mediaFileId) {
|
||||
List<MediaFaceRegionEntity> regions = repository.findByMediaFileId(mediaFileId);
|
||||
if (regions.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
Set<Long> personIds = regions.stream().map(MediaFaceRegionEntity::getPersonId).filter(java.util.Objects::nonNull).collect(Collectors.toSet());
|
||||
Map<Long, String> namesById = personIds.isEmpty() ? Map.of()
|
||||
: java.util.stream.StreamSupport.stream(personRepository.findAllById(personIds).spliterator(), false)
|
||||
.filter(person -> person.getName() != null)
|
||||
.collect(Collectors.toMap(PersonEntity::getId, PersonEntity::getName));
|
||||
return regions.stream()
|
||||
.map(region -> new DisplayRegion(EEntityKind.valueOf(region.getKind()), region.getLabel(),
|
||||
region.getPersonId() == null ? null : namesById.get(region.getPersonId())))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/** One row for {@code MediaInfoPane}'s persons/animals chips — {@code personName} is {@code null} for an unnamed cluster or any {@link EEntityKind#ANIMAL}. */
|
||||
public record DisplayRegion(EEntityKind kind, @Nullable String label, @Nullable String personName) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces every region {@code mediaFileId} previously had with {@code regions} — delete-then-reinsert,
|
||||
* the same convention {@code LibraryFolderService.persistTags} uses for {@code media_file_tag}, since a
|
||||
* re-detection's region set is typically small and unrelated row-by-row diffing buys nothing.
|
||||
*
|
||||
* <p>Every inserted row starts unlinked ({@code personId = null}) and unconfirmed — see
|
||||
* {@link MediaFaceRegion#confirmed()}'s own javadoc for why automatic detection never confirms a region
|
||||
* itself.
|
||||
*/
|
||||
public void replaceRegionsFor(Long mediaFileId, List<DetectedRegion> regions) {
|
||||
repository.deleteByMediaFileIdIn(List.of(mediaFileId));
|
||||
if (regions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Instant now = Instant.now();
|
||||
List<MediaFaceRegionEntity> entities = regions.stream().map(region -> toEntity(mediaFileId, region, now)).toList();
|
||||
repository.saveAll(entities);
|
||||
}
|
||||
|
||||
private static MediaFaceRegionEntity toEntity(Long mediaFileId, DetectedRegion region, Instant detectedAt) {
|
||||
return MediaFaceRegionEntity.builder()
|
||||
.mediaFileId(mediaFileId)
|
||||
.personId(null)
|
||||
.kind(region.kind().name())
|
||||
.areaX(region.box().x())
|
||||
.areaY(region.box().y())
|
||||
.areaW(region.box().w())
|
||||
.areaH(region.box().h())
|
||||
.confidence(region.confidence())
|
||||
.embedding(EmbeddingCodec.toBytes(region.embedding()))
|
||||
.label(region.label())
|
||||
.sourceProvider(region.sourceProvider())
|
||||
.confirmed(false)
|
||||
.detectedAt(detectedAt)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.RecognitionResult;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
|
||||
/**
|
||||
* Finds faces and animals in one already-decoded photo — the local/remote provider seam, mirroring
|
||||
* {@code IPlaceSearchService}. {@link LocalRecognitionService} answers entirely offline, delegating to
|
||||
* whichever {@code engine} package implementation is wired (DJL/ONNX today, swappable later without
|
||||
* touching this seam); {@link RecognitionService} is the {@code @Primary} bean actually injected everywhere,
|
||||
* routing each call to either the local engines or a user-configured remote provider (see its own javadoc)
|
||||
* so the caller never needs to know which one actually answered.
|
||||
*/
|
||||
public interface IRecognitionService {
|
||||
|
||||
RecognitionResult analyze(BufferedImage image);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.icroco.pholio.domain.recognition.RecognitionResult;
|
||||
import org.icroco.pholio.infra.recognition.engine.IFaceDetectionEngine;
|
||||
import org.icroco.pholio.infra.recognition.engine.IObjectDetectionEngine;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link IRecognitionService} answered entirely by the local engines — no network involved. The class name
|
||||
* itself is the "runs locally" signal {@link RecognitionService}'s own javadoc refers to, the same
|
||||
* convention {@code LocalPlaceSearchService} uses for geocoding.
|
||||
*/
|
||||
@Component
|
||||
public class LocalRecognitionService implements IRecognitionService {
|
||||
|
||||
private final IFaceDetectionEngine faceEngine;
|
||||
private final IObjectDetectionEngine animalEngine;
|
||||
|
||||
public LocalRecognitionService(IFaceDetectionEngine faceEngine, IObjectDetectionEngine animalEngine) {
|
||||
this.faceEngine = faceEngine;
|
||||
this.animalEngine = animalEngine;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecognitionResult analyze(BufferedImage image) {
|
||||
List<DetectedRegion> regions = new ArrayList<>(faceEngine.detectFaces(image));
|
||||
regions.addAll(animalEngine.detectAnimals(image));
|
||||
return new RecognitionResult(regions);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
|
||||
import org.icroco.pholio.infra.library.LibraryFolderService;
|
||||
import org.icroco.pholio.infra.persistence.folder.MediaFileEntity;
|
||||
import org.icroco.pholio.infra.persistence.folder.MediaFileMapper;
|
||||
import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.icroco.pholio.infra.scheduling.IStartupTask;
|
||||
import org.icroco.pholio.infra.task.TaskService;
|
||||
import org.icroco.pholio.infra.task.TaskType;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.DependsOn;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Backfills face/animal detection, at every application start, for every {@code media_file} row in the
|
||||
* currently open library still missing {@link EMediaFileProcessingFlag#FACE_DETECTED} — the library-wide
|
||||
* catch-up counterpart to {@link MediaFileRecognitionTrigger}'s per-import/per-sync hook, for files that
|
||||
* predate this feature or whose earlier detection attempt never completed.
|
||||
*
|
||||
* <p>Unlike {@code MediaFileProcessingFlagsBackfillTask} (a cheap flag-only fix), this dispatches real
|
||||
* detection work, so it is submitted as a single, <em>visible</em> {@code TaskType.IMAGE_ANALYSIS} batch
|
||||
* (not silent) — the "N files remaining" progress this feature is explicitly meant to show in the status
|
||||
* bar, rather than a silent background fix.
|
||||
*
|
||||
* <p>{@code @Order(1)}: runs after {@code MediaFileProcessingFlagsBackfillTask} (@Order(0)), so this reads
|
||||
* flags that backfill has already reconciled with reality.
|
||||
*
|
||||
* <p>{@code @DependsOn("libraryService")} for the same reason as that task: {@code media_file} lives in the
|
||||
* per-library routing datasource, which must already point at an open library.
|
||||
*/
|
||||
@Component
|
||||
@DependsOn("libraryService")
|
||||
@Order(1)
|
||||
public class MediaFaceDetectionBackfillTask implements IStartupTask {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MediaFaceDetectionBackfillTask.class);
|
||||
|
||||
private final AppPreferences preferences;
|
||||
private final MediaFileRepository mediaFileRepository;
|
||||
private final MediaFileMapper mediaFileMapper;
|
||||
private final LibraryFolderService libraryFolderService;
|
||||
private final MediaRecognitionService mediaRecognitionService;
|
||||
private final TaskService taskService;
|
||||
|
||||
public MediaFaceDetectionBackfillTask(AppPreferences preferences, MediaFileRepository mediaFileRepository,
|
||||
MediaFileMapper mediaFileMapper, LibraryFolderService libraryFolderService,
|
||||
MediaRecognitionService mediaRecognitionService, TaskService taskService) {
|
||||
this.preferences = preferences;
|
||||
this.mediaFileRepository = mediaFileRepository;
|
||||
this.mediaFileMapper = mediaFileMapper;
|
||||
this.libraryFolderService = libraryFolderService;
|
||||
this.mediaRecognitionService = mediaRecognitionService;
|
||||
this.taskService = taskService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean shouldRun() {
|
||||
return preferences.getValueOr("recognition", "enabled", Boolean.class, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
List<MediaFileEntity> pending = mediaFileRepository.findAll().stream()
|
||||
.filter(entity -> !mediaFileMapper.toDomain(entity)
|
||||
.hasProcessingFlag(EMediaFileProcessingFlag.FACE_DETECTED))
|
||||
.toList();
|
||||
if (pending.isEmpty()) {
|
||||
log.debug("No media file pending face/animal detection");
|
||||
return;
|
||||
}
|
||||
TaskService.BatchTask batch = taskService.submitBatch(TaskType.IMAGE_ANALYSIS, "Detecting faces & animals", pending.size(), false);
|
||||
pending.forEach(entity -> {
|
||||
Long mediaFileId = Objects.requireNonNull(entity.getId(), "A persisted MediaFileEntity always has an id");
|
||||
libraryFolderService.absolutePathOf(mediaFileMapper.toDomain(entity))
|
||||
.ifPresentOrElse(
|
||||
absolute -> taskService.execute(TaskType.IMAGE_ANALYSIS,
|
||||
() -> mediaRecognitionService.detect(mediaFileId, absolute, batch)),
|
||||
batch::completedOne);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
|
||||
import org.icroco.pholio.domain.library.MediaFile;
|
||||
import org.icroco.pholio.infra.library.LibraryFolderService;
|
||||
import org.icroco.pholio.infra.library.MediaFileAnalyzedEvent;
|
||||
import org.icroco.pholio.infra.persistence.folder.MediaFileMapper;
|
||||
import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.icroco.pholio.infra.task.TaskService;
|
||||
import org.icroco.pholio.infra.task.TaskType;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Runs face/animal detection for a media file right after {@code MediaAnalysisService} finishes its own
|
||||
* thumbnail+phash pass — covers both a freshly imported file and a re-synced, modified one, since
|
||||
* {@code LibraryFolderService.reimportModifiedFile} already calls {@code generateThumbnail} unconditionally,
|
||||
* whose {@link MediaFileAnalyzedEvent} this listens to either way. No separate hook into
|
||||
* {@code SyncReportService} is needed.
|
||||
*
|
||||
* <p>Skips already-detected files whose pixels are unchanged ({@link MediaFileAnalyzedEvent#pixelsLikelyUnchanged()}
|
||||
* — a metadata-only edit, e.g. a rating change, re-fires this event but must not re-run detection.
|
||||
*/
|
||||
@Component
|
||||
public class MediaFileRecognitionTrigger {
|
||||
|
||||
private final AppPreferences preferences;
|
||||
private final MediaFileRepository mediaFileRepository;
|
||||
private final MediaFileMapper mediaFileMapper;
|
||||
private final LibraryFolderService libraryFolderService;
|
||||
private final MediaRecognitionService mediaRecognitionService;
|
||||
private final TaskService taskService;
|
||||
|
||||
public MediaFileRecognitionTrigger(AppPreferences preferences, MediaFileRepository mediaFileRepository,
|
||||
MediaFileMapper mediaFileMapper, LibraryFolderService libraryFolderService,
|
||||
MediaRecognitionService mediaRecognitionService, TaskService taskService) {
|
||||
this.preferences = preferences;
|
||||
this.mediaFileRepository = mediaFileRepository;
|
||||
this.mediaFileMapper = mediaFileMapper;
|
||||
this.libraryFolderService = libraryFolderService;
|
||||
this.mediaRecognitionService = mediaRecognitionService;
|
||||
this.taskService = taskService;
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void onAnalyzed(MediaFileAnalyzedEvent event) {
|
||||
if (!preferences.getValueOr("recognition", "enabled", Boolean.class, true)) {
|
||||
return;
|
||||
}
|
||||
mediaFileRepository.findById(event.mediaFileId()).ifPresent(entity -> {
|
||||
MediaFile mediaFile = mediaFileMapper.toDomain(entity);
|
||||
if (mediaFile.hasProcessingFlag(EMediaFileProcessingFlag.FACE_DETECTED) && event.pixelsLikelyUnchanged()) {
|
||||
return;
|
||||
}
|
||||
Long mediaFileId = Objects.requireNonNull(entity.getId(), "A persisted MediaFileEntity always has an id");
|
||||
libraryFolderService.absolutePathOf(mediaFile).ifPresent(absolute -> {
|
||||
TaskService.BatchTask unit = taskService.submitBatch(TaskType.IMAGE_ANALYSIS, "Detecting faces & animals", 1, true);
|
||||
taskService.execute(TaskType.IMAGE_ANALYSIS, () -> mediaRecognitionService.detect(mediaFileId, absolute, unit));
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
/**
|
||||
* Published once {@code MediaRecognitionService.detect} has replaced a {@code MediaFile}'s detected
|
||||
* regions — {@code MediaInfoPane}'s persons/animals row listens for this to refresh if the file it is
|
||||
* currently showing is the one just analyzed.
|
||||
*/
|
||||
public record MediaFileRecognizedEvent(Long mediaFileId) {
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
|
||||
import org.icroco.pholio.domain.library.MediaFile;
|
||||
import org.icroco.pholio.domain.media.ImageFormat;
|
||||
import org.icroco.pholio.domain.recognition.RecognitionResult;
|
||||
import org.icroco.pholio.infra.media.MediaFormatRegistry;
|
||||
import org.icroco.pholio.infra.media.ThumbnailGenerator;
|
||||
import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
|
||||
import org.icroco.pholio.infra.task.TaskService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.annotation.DependsOn;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* The per-file face/animal detection pipeline — decode, analyze, persist, cluster, flag, notify. Run off
|
||||
* {@code TaskType.IMAGE_ANALYSIS} by both {@link MediaFileRecognitionTrigger} (import/re-sync) and
|
||||
* {@link MediaFaceDetectionBackfillTask} (startup backlog).
|
||||
*
|
||||
* <p>Decodes the original file rather than reusing the small cached thumbnail, for accuracy on small/distant
|
||||
* faces — more expensive than the thumbnail path {@code MediaAnalysisService} optimizes for reuse; see the
|
||||
* implementation plan's own risk notes if this needs revisiting against very large libraries.
|
||||
*
|
||||
* <p>{@code @DependsOn("libraryService")} for the same reason as {@code MediaAnalysisService}: the routing
|
||||
* datasource must already point at an open library before this touches a repository.
|
||||
*/
|
||||
@Service
|
||||
@DependsOn("libraryService")
|
||||
public class MediaRecognitionService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MediaRecognitionService.class);
|
||||
|
||||
private final MediaFormatRegistry formats;
|
||||
private final ThumbnailGenerator thumbnailGenerator;
|
||||
private final IRecognitionService recognitionService;
|
||||
private final FaceRegionQueryService faceRegionQueryService;
|
||||
private final FaceClusteringService faceClusteringService;
|
||||
private final MediaFileRepository mediaFileRepository;
|
||||
private final ApplicationEventPublisher publisher;
|
||||
|
||||
public MediaRecognitionService(MediaFormatRegistry formats, ThumbnailGenerator thumbnailGenerator,
|
||||
IRecognitionService recognitionService, FaceRegionQueryService faceRegionQueryService,
|
||||
FaceClusteringService faceClusteringService, MediaFileRepository mediaFileRepository,
|
||||
ApplicationEventPublisher publisher) {
|
||||
this.formats = formats;
|
||||
this.thumbnailGenerator = thumbnailGenerator;
|
||||
this.recognitionService = recognitionService;
|
||||
this.faceRegionQueryService = faceRegionQueryService;
|
||||
this.faceClusteringService = faceClusteringService;
|
||||
this.mediaFileRepository = mediaFileRepository;
|
||||
this.publisher = publisher;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param batchTask completed exactly once, in a {@code finally}, whatever the outcome — the same
|
||||
* discipline {@code MediaAnalysisService.hashAndPublish} follows for its own batch.
|
||||
*/
|
||||
public void detect(Long mediaFileId, Path absolute, TaskService.BatchTask batchTask) {
|
||||
try {
|
||||
Optional<ImageFormat> format = formats.formatOf(absolute);
|
||||
if (format.isEmpty()) {
|
||||
log.warn("'{}' is no longer a recognised format; skipping recognition", absolute);
|
||||
return;
|
||||
}
|
||||
Optional<BufferedImage> decoded = thumbnailGenerator.decode(absolute, format.get());
|
||||
if (decoded.isEmpty()) {
|
||||
log.debug("No pixels obtainable for '{}'; no recognition run", absolute);
|
||||
return;
|
||||
}
|
||||
BufferedImage oriented = thumbnailGenerator.applyOrientation(decoded.get(), thumbnailGenerator.orientationOf(absolute));
|
||||
|
||||
RecognitionResult result = recognitionService.analyze(oriented);
|
||||
faceRegionQueryService.replaceRegionsFor(mediaFileId, result.regions());
|
||||
faceClusteringService.clusterUnnamedPersons();
|
||||
|
||||
mediaFileRepository.findById(mediaFileId).ifPresent(entity -> {
|
||||
entity.setProcessingFlags(MediaFile.withBit(entity.getProcessingFlags(), EMediaFileProcessingFlag.FACE_DETECTED));
|
||||
mediaFileRepository.save(entity);
|
||||
});
|
||||
publisher.publishEvent(new MediaFileRecognizedEvent(mediaFileId));
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
log.warn("Face/animal recognition failed unexpectedly for '{}': {}", absolute, e.toString(), e);
|
||||
}
|
||||
finally {
|
||||
batchTask.completedOne();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
/**
|
||||
* One user-configured remote recognition provider, as stored (a JSON list of these) in the
|
||||
* {@code recognition.providers-json} preference — see {@link RecognitionService} for how it is picked and
|
||||
* used, and {@link ERecognitionProviderKind} for the wire contract it must speak.
|
||||
*
|
||||
* @param name shown in the "active provider" picker; also the key {@code recognition.active-provider}
|
||||
* stores to select this config
|
||||
* @param urlTemplate the endpoint to {@code POST} the analysis request to
|
||||
* @param apiKey sent as-is in the request body; the provider validates it however it wants
|
||||
* @param kind which wire contract to speak — see {@link ERecognitionProviderKind}
|
||||
*/
|
||||
public record RecognitionProviderConfig(String name, String urlTemplate, String apiKey, ERecognitionProviderKind kind) {
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package org.icroco.pholio.infra.recognition;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.BoundingBox;
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.domain.recognition.RecognitionResult;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The {@code @Primary} {@link IRecognitionService} — the one bean actually injected wherever face/animal
|
||||
* recognition is needed. Routes each call to either {@link LocalRecognitionService} or a user-configured
|
||||
* remote provider, transparently to the caller — the exact same shape {@code PlaceSearchService} uses for
|
||||
* geocoding, just a strict proxy rather than a merge: a remote provider's answer is used as-is, never
|
||||
* combined with the local engines' own.
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code recognition.active-provider} blank/unset (the default) → {@link LocalRecognitionService},
|
||||
* always.
|
||||
* <li>otherwise the matching {@link RecognitionProviderConfig} from {@code recognition.providers-json} is
|
||||
* called over HTTP, per {@link ERecognitionProviderKind}'s wire contract.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Falls back to {@link LocalRecognitionService} whenever the configured provider cannot answer — an
|
||||
* unknown/deleted provider name, a non-2xx response, a network failure, a malformed reply — rather than
|
||||
* surfacing an error: local detection is always available, so the background pipeline keeps working offline
|
||||
* the same way it always has.
|
||||
*/
|
||||
@Component
|
||||
@Primary
|
||||
public class RecognitionService implements IRecognitionService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(RecognitionService.class);
|
||||
|
||||
private static final String PREFERENCE_GROUP = "recognition";
|
||||
private static final String ACTIVE_PROVIDER_KEY = "active-provider";
|
||||
private static final String PROVIDERS_KEY = "providers-json";
|
||||
|
||||
private final LocalRecognitionService local;
|
||||
private final AppPreferences preferences;
|
||||
private final ObjectMapper json = new ObjectMapper();
|
||||
|
||||
private final HttpClient httpClient = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofSeconds(5))
|
||||
.build();
|
||||
|
||||
public RecognitionService(LocalRecognitionService local, AppPreferences preferences) {
|
||||
this.local = local;
|
||||
this.preferences = preferences;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecognitionResult analyze(BufferedImage image) {
|
||||
String activeProviderName = preferences.text(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY).orElse("");
|
||||
if (activeProviderName.isBlank()) {
|
||||
log.debug("No active recognition provider configured, analyzing locally");
|
||||
return local.analyze(image);
|
||||
}
|
||||
log.debug("Active recognition provider is '{}', analyzing remotely", activeProviderName);
|
||||
return providers().stream()
|
||||
.filter(provider -> provider.name().equals(activeProviderName))
|
||||
.findFirst()
|
||||
.map(provider -> analyzeRemote(provider, image))
|
||||
.orElseGet(() -> {
|
||||
log.warn("Active recognition provider '{}' is no longer configured, falling back to local analysis",
|
||||
activeProviderName);
|
||||
return local.analyze(image);
|
||||
});
|
||||
}
|
||||
|
||||
/** The configured provider list, in the order they were added — never {@code null}. Exposed for the Maintenance recognition tab. */
|
||||
public List<RecognitionProviderConfig> providers() {
|
||||
String raw = preferences.text(PREFERENCE_GROUP, PROVIDERS_KEY).orElse("[]");
|
||||
try {
|
||||
return json.readValue(raw, new TypeReference<List<RecognitionProviderConfig>>() {});
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
log.warn("Could not read recognition.providers-json ('{}') as a provider list, treating it as empty", raw, e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes {@code providers} back to {@code recognition.providers-json} — the Maintenance tab's Save action. */
|
||||
public void saveProviders(List<RecognitionProviderConfig> providers) {
|
||||
preferences.setValue(PREFERENCE_GROUP, PROVIDERS_KEY, json.writeValueAsString(providers));
|
||||
}
|
||||
|
||||
/** The active provider's name, or blank for "local only". */
|
||||
public String activeProvider() {
|
||||
return preferences.text(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY).orElse("");
|
||||
}
|
||||
|
||||
/** Writes {@code providerName} (blank for "local only") to {@code recognition.active-provider}. */
|
||||
public void saveActiveProvider(String providerName) {
|
||||
preferences.setValue(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY, providerName);
|
||||
}
|
||||
|
||||
private RecognitionResult analyzeRemote(RecognitionProviderConfig provider, BufferedImage image) {
|
||||
log.info("Calling recognition provider '{}' (kind={})", provider.name(), provider.kind());
|
||||
try {
|
||||
String body = requestBody(provider, image);
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create(provider.urlTemplate()))
|
||||
.header("Content-Type", "application/json")
|
||||
.timeout(Duration.ofSeconds(20))
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8))
|
||||
.build();
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
log.info("Recognition provider '{}' responded HTTP {}", provider.name(), response.statusCode());
|
||||
if (response.statusCode() != 200) {
|
||||
log.warn("Recognition provider '{}' returned HTTP {}, falling back to local analysis",
|
||||
provider.name(), response.statusCode());
|
||||
return local.analyze(image);
|
||||
}
|
||||
RecognitionResult result = parse(response.body(), provider.name());
|
||||
log.info("Recognition provider '{}' returned {} region(s)", provider.name(), result.regions().size());
|
||||
return result;
|
||||
}
|
||||
catch (IOException e) {
|
||||
log.warn("Could not reach recognition provider '{}', falling back to local analysis", provider.name(), e);
|
||||
return local.analyze(image);
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return local.analyze(image);
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
// A remote service changing shape, or simply misbehaving, must not crash the recognition pipeline.
|
||||
log.warn("Could not parse recognition provider '{}' response, falling back to local analysis", provider.name(), e);
|
||||
return local.analyze(image);
|
||||
}
|
||||
}
|
||||
|
||||
private String requestBody(RecognitionProviderConfig provider, BufferedImage image) throws IOException {
|
||||
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", buffer);
|
||||
ObjectNode node = json.createObjectNode();
|
||||
node.put("apiKey", provider.apiKey());
|
||||
node.put("width", image.getWidth());
|
||||
node.put("height", image.getHeight());
|
||||
node.put("imageBase64", Base64.getEncoder().encodeToString(buffer.toByteArray()));
|
||||
return json.writeValueAsString(node);
|
||||
}
|
||||
|
||||
private RecognitionResult parse(String body, String providerName) {
|
||||
JsonNode root = json.readTree(body);
|
||||
List<DetectedRegion> regions = new ArrayList<>();
|
||||
for (JsonNode node : root) {
|
||||
regions.add(toDetectedRegion(node, providerName));
|
||||
}
|
||||
return new RecognitionResult(regions);
|
||||
}
|
||||
|
||||
private static DetectedRegion toDetectedRegion(JsonNode node, String providerName) {
|
||||
EEntityKind kind = EEntityKind.valueOf(node.path("kind").asString("PERSON"));
|
||||
BoundingBox box = new BoundingBox(node.path("x").asDouble(), node.path("y").asDouble(),
|
||||
node.path("w").asDouble(), node.path("h").asDouble());
|
||||
return new DetectedRegion(kind, box, node.path("confidence").asDouble(1.0),
|
||||
null, node.path("label").asString(null), providerName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package org.icroco.pholio.infra.recognition.engine;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The local face-detection/embedding engine seam — independent of {@code IRecognitionService}'s own
|
||||
* local-vs-remote seam, so swapping this (DJL/ONNX today, JavaCV/OpenCV DNN or anything else later) never
|
||||
* touches {@code RecognitionService}'s routing logic. Each returned {@link DetectedRegion} carries an
|
||||
* embedding for {@code FaceClusteringService} to group across photos.
|
||||
*/
|
||||
public interface IFaceDetectionEngine {
|
||||
|
||||
List<DetectedRegion> detectFaces(BufferedImage image);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package org.icroco.pholio.infra.recognition.engine;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The local animal-detection engine seam — see {@link IFaceDetectionEngine}'s own javadoc for why this is
|
||||
* kept separate from {@code IRecognitionService}'s local-vs-remote seam. Each returned {@link DetectedRegion}
|
||||
* carries a species {@code label} (e.g. {@code "dog"}), not an embedding — individual animal identity is not
|
||||
* attempted in this iteration.
|
||||
*
|
||||
* <p>Named for animals specifically rather than "objects" in general: generic object recognition is a
|
||||
* separate, later phase, and will get its own method here (or a sibling interface) once designed, without
|
||||
* needing to touch any existing caller.
|
||||
*/
|
||||
public interface IObjectDetectionEngine {
|
||||
|
||||
List<DetectedRegion> detectAnimals(BufferedImage image);
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package org.icroco.pholio.infra.recognition.engine;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.List;
|
||||
|
||||
/** Placeholder {@link IObjectDetectionEngine} — see {@link NoopFaceDetectionEngine}'s own javadoc. */
|
||||
public class NoopAnimalDetectionEngine implements IObjectDetectionEngine {
|
||||
|
||||
@Override
|
||||
public List<DetectedRegion> detectAnimals(BufferedImage image) {
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package org.icroco.pholio.infra.recognition.engine;
|
||||
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Placeholder {@link IFaceDetectionEngine} — always finds nothing. Superseded by
|
||||
* {@code YuNetSFaceFaceDetectionEngine} as the real {@code @Component}; kept as a plain (non-Spring) class
|
||||
* for tests that need a face engine stand-in without loading actual ONNX models.
|
||||
*/
|
||||
public class NoopFaceDetectionEngine implements IFaceDetectionEngine {
|
||||
|
||||
@Override
|
||||
public List<DetectedRegion> detectFaces(BufferedImage image) {
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
/** Greedy IoU-based NMS, shared by every detector in this package — no per-class grouping (callers that need it group first). */
|
||||
final class NonMaxSuppression {
|
||||
|
||||
private NonMaxSuppression() {
|
||||
}
|
||||
|
||||
record Box(double x1, double y1, double x2, double y2, double score, int index) {
|
||||
}
|
||||
|
||||
/** {@code box.index()} of every survivor, highest score first. */
|
||||
static List<Integer> suppress(List<Box> boxes, double iouThreshold) {
|
||||
List<Box> byScoreDesc = boxes.stream().sorted(Comparator.comparingDouble(Box::score).reversed()).toList();
|
||||
List<Box> kept = new ArrayList<>();
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (Box candidate : byScoreDesc) {
|
||||
boolean overlapsKept = kept.stream().anyMatch(k -> iou(candidate, k) > iouThreshold);
|
||||
if (!overlapsKept) {
|
||||
kept.add(candidate);
|
||||
result.add(candidate.index());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static double iou(Box a, Box b) {
|
||||
double x1 = Math.max(a.x1(), b.x1());
|
||||
double y1 = Math.max(a.y1(), b.y1());
|
||||
double x2 = Math.min(a.x2(), b.x2());
|
||||
double y2 = Math.min(a.y2(), b.y2());
|
||||
double inter = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
|
||||
double areaA = (a.x2() - a.x1()) * (a.y2() - a.y1());
|
||||
double areaB = (b.x2() - b.x1()) * (b.y2() - b.y1());
|
||||
return inter / (areaA + areaB - inter + 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
|
||||
/** NCHW float tensor extraction from a {@link BufferedImage} — shared by every engine in this package. */
|
||||
final class OnnxImageTensors {
|
||||
|
||||
private OnnxImageTensors() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw (unnormalized, 0-255) pixel values in planar NCHW layout — every model bundled here was exported
|
||||
* from OpenCV/PyTorch pipelines that feed raw pixel floats, never {@code /255}-scaled ones.
|
||||
*
|
||||
* @param swapToBgr {@code true} to write the R/G/B planes in B,G,R order — every bundled model here was
|
||||
* trained against OpenCV's native BGR channel order except SFace, which explicitly
|
||||
* swaps to RGB before its own forward pass (see {@code YuNetSFaceFaceDetectionEngine}).
|
||||
* {@link BufferedImage#getRGB} always hands back R/G/B regardless of the source file's
|
||||
* own encoding, so this is the one place that channel order is ever chosen.
|
||||
*/
|
||||
static float[] toChwFloats(BufferedImage image, boolean swapToBgr) {
|
||||
int width = image.getWidth();
|
||||
int height = image.getHeight();
|
||||
int plane = width * height;
|
||||
float[] data = new float[3 * plane];
|
||||
for (int y = 0; y < height; y++) {
|
||||
for (int x = 0; x < width; x++) {
|
||||
int rgb = image.getRGB(x, y);
|
||||
int r = (rgb >> 16) & 0xFF;
|
||||
int g = (rgb >> 8) & 0xFF;
|
||||
int b = rgb & 0xFF;
|
||||
int idx = y * width + x;
|
||||
if (swapToBgr) {
|
||||
data[idx] = b;
|
||||
data[plane + idx] = g;
|
||||
data[2 * plane + idx] = r;
|
||||
}
|
||||
else {
|
||||
data[idx] = r;
|
||||
data[plane + idx] = g;
|
||||
data[2 * plane + idx] = b;
|
||||
}
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import ai.onnxruntime.OrtEnvironment;
|
||||
import ai.onnxruntime.OrtException;
|
||||
import ai.onnxruntime.OrtSession;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
|
||||
/**
|
||||
* Opens an {@link OrtSession} for a bundled ONNX model — packaged as a classpath resource under
|
||||
* {@code /models/recognition/}, since ONNX Runtime's Java API needs a real file path (or byte array; a file
|
||||
* path is what lets ONNX Runtime memory-map the weights instead of holding a second copy in the JVM heap).
|
||||
* The resource is extracted to the OS temp directory once and reused on every later call/run — its filename
|
||||
* alone is the cache key, since these bundled models never change without a Pholio version bump.
|
||||
*/
|
||||
final class OnnxModelLoader {
|
||||
|
||||
private static final OrtEnvironment ENVIRONMENT = OrtEnvironment.getEnvironment();
|
||||
|
||||
private OnnxModelLoader() {
|
||||
}
|
||||
|
||||
static OrtSession load(String classpathResource) {
|
||||
try {
|
||||
Path modelFile = extractToCache(classpathResource);
|
||||
return ENVIRONMENT.createSession(modelFile.toString(), new OrtSession.SessionOptions());
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new UncheckedIOException("Could not extract bundled ONNX model '" + classpathResource + "'", e);
|
||||
}
|
||||
catch (OrtException e) {
|
||||
throw new IllegalStateException("Could not open ONNX session for '" + classpathResource + "'", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static Path extractToCache(String classpathResource) throws IOException {
|
||||
Path cacheDir = Path.of(System.getProperty("java.io.tmpdir"), "pholio-recognition-models");
|
||||
Files.createDirectories(cacheDir);
|
||||
String fileName = classpathResource.substring(classpathResource.lastIndexOf('/') + 1);
|
||||
Path target = cacheDir.resolve(fileName);
|
||||
if (Files.exists(target)) {
|
||||
return target;
|
||||
}
|
||||
try (InputStream in = OnnxModelLoader.class.getResourceAsStream(classpathResource)) {
|
||||
if (in == null) {
|
||||
throw new IOException("Missing bundled model resource: " + classpathResource);
|
||||
}
|
||||
Path staging = Files.createTempFile(cacheDir, "extract-", ".onnx");
|
||||
Files.copy(in, staging, StandardCopyOption.REPLACE_EXISTING);
|
||||
Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
/**
|
||||
* The least-squares 2D similarity transform (uniform scale + rotation + translation, no shear/reflection)
|
||||
* mapping {@code src} points onto {@code dst} points — used to align a detected face's 5 landmarks onto
|
||||
* SFace's fixed reference layout before cropping. Equivalent to Umeyama's algorithm restricted to the
|
||||
* pure-similarity case, but solved directly as a complex-number linear regression rather than via an SVD:
|
||||
* treating each 2D point as a complex number {@code p = x + iy}, the best-fit {@code dst ≈ a*src + t} (a, t
|
||||
* complex) minimizing squared error has the closed form below — no matrix decomposition needed.
|
||||
*/
|
||||
final class SimilarityTransform {
|
||||
|
||||
private SimilarityTransform() {
|
||||
}
|
||||
|
||||
/** {@code {aRe, aIm, tRe, tIm}} such that {@code dst ≈ a*src + t} with {@code a = aRe + i*aIm}, {@code t = tRe + i*tIm}. */
|
||||
static double[] estimate(double[][] src, double[][] dst) {
|
||||
int n = src.length;
|
||||
double meanSrcX = 0, meanSrcY = 0, meanDstX = 0, meanDstY = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
meanSrcX += src[i][0];
|
||||
meanSrcY += src[i][1];
|
||||
meanDstX += dst[i][0];
|
||||
meanDstY += dst[i][1];
|
||||
}
|
||||
meanSrcX /= n;
|
||||
meanSrcY /= n;
|
||||
meanDstX /= n;
|
||||
meanDstY /= n;
|
||||
|
||||
double numRe = 0, numIm = 0, den = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
double sx = src[i][0] - meanSrcX;
|
||||
double sy = src[i][1] - meanSrcY;
|
||||
double dx = dst[i][0] - meanDstX;
|
||||
double dy = dst[i][1] - meanDstY;
|
||||
// conj(s) * d = (sx - i*sy)(dx + i*dy) = (sx*dx + sy*dy) + i*(sx*dy - sy*dx)
|
||||
numRe += sx * dx + sy * dy;
|
||||
numIm += sx * dy - sy * dx;
|
||||
den += sx * sx + sy * sy;
|
||||
}
|
||||
double aRe = numRe / den;
|
||||
double aIm = numIm / den;
|
||||
double tRe = meanDstX - (aRe * meanSrcX - aIm * meanSrcY);
|
||||
double tIm = meanDstY - (aIm * meanSrcX + aRe * meanSrcY);
|
||||
return new double[]{ aRe, aIm, tRe, tIm };
|
||||
}
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import ai.onnxruntime.OnnxTensor;
|
||||
import ai.onnxruntime.OrtEnvironment;
|
||||
import ai.onnxruntime.OrtException;
|
||||
import ai.onnxruntime.OrtSession;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.icroco.pholio.domain.recognition.BoundingBox;
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.infra.recognition.engine.IObjectDetectionEngine;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.nio.FloatBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* {@link IObjectDetectionEngine} backed by <a href="https://github.com/Megvii-BaseDetection/YOLOX">YOLOX-Nano</a>
|
||||
* (Apache-2.0, Megvii), a general 80-class COCO detector filtered down to the animal classes — species label
|
||||
* only, no individual animal identity in this iteration. Preprocessing (letterbox, gray padding) and
|
||||
* postprocessing (grid/stride decode, sigmoid already baked into the exported graph) follow YOLOX's own
|
||||
* {@code demo/ONNXRuntime/onnx_inference.py} and {@code yolox/data/data_augment.py} exactly; input/output
|
||||
* tensor names and shape were confirmed directly against the bundled {@code .onnx} file via
|
||||
* {@code OrtSession.getInputInfo()}/{@code getOutputInfo()}.
|
||||
*/
|
||||
@Component
|
||||
public class YoloXAnimalDetectionEngine implements IObjectDetectionEngine, AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(YoloXAnimalDetectionEngine.class);
|
||||
|
||||
private static final String SOURCE_PROVIDER = "local-onnx";
|
||||
|
||||
private static final int INPUT_SIZE = 416;
|
||||
private static final int[] STRIDES = { 8, 16, 32 };
|
||||
/** YOLOX-Nano's own {@code demo/ONNXRuntime} defaults. */
|
||||
private static final double SCORE_THRESHOLD = 0.3;
|
||||
private static final double NMS_THRESHOLD = 0.45;
|
||||
|
||||
/** COCO's 80 class names, official order — only indices {@link #ANIMAL_CLASS_MIN}..{@link #ANIMAL_CLASS_MAX} are ever looked at. */
|
||||
private static final String[] COCO_CLASSES = {
|
||||
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
|
||||
"fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
|
||||
"elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
|
||||
"skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
|
||||
"tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
|
||||
"sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
|
||||
"potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
|
||||
"microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
|
||||
"hair drier", "toothbrush"
|
||||
};
|
||||
private static final int ANIMAL_CLASS_MIN = 14; // bird
|
||||
private static final int ANIMAL_CLASS_MAX = 23; // giraffe
|
||||
|
||||
private final OrtEnvironment environment = OrtEnvironment.getEnvironment();
|
||||
private final OrtSession session;
|
||||
|
||||
public YoloXAnimalDetectionEngine() {
|
||||
this.session = OnnxModelLoader.load("/models/recognition/yolox_nano.onnx");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DetectedRegion> detectAnimals(BufferedImage image) {
|
||||
try {
|
||||
List<RawDetection> detections = detectRaw(image);
|
||||
List<DetectedRegion> regions = new ArrayList<>(detections.size());
|
||||
for (RawDetection d : detections) {
|
||||
double boxWidth = d.x2() - d.x1();
|
||||
double boxHeight = d.y2() - d.y1();
|
||||
BoundingBox box = new BoundingBox((d.x1() + boxWidth / 2) / image.getWidth(),
|
||||
(d.y1() + boxHeight / 2) / image.getHeight(),
|
||||
boxWidth / image.getWidth(), boxHeight / image.getHeight());
|
||||
regions.add(new DetectedRegion(EEntityKind.ANIMAL, box, d.score(), null, d.label(), SOURCE_PROVIDER));
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
catch (OrtException e) {
|
||||
log.warn("Animal detection failed unexpectedly: {}", e.toString(), e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private record RawDetection(double x1, double y1, double x2, double y2, double score, String label) {
|
||||
}
|
||||
|
||||
private List<RawDetection> detectRaw(BufferedImage image) throws OrtException {
|
||||
Letterbox letterbox = letterbox(image, INPUT_SIZE);
|
||||
float[] chw = OnnxImageTensors.toChwFloats(letterbox.image(), true); // YOLOX expects BGR, raw 0-255 (no /255 normalization)
|
||||
|
||||
List<RawDetection> candidates;
|
||||
try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, INPUT_SIZE, INPUT_SIZE });
|
||||
OrtSession.Result result = session.run(Map.of("images", input))) {
|
||||
float[][][] output = (float[][][]) result.get("output").orElseThrow().getValue();
|
||||
candidates = decode(output[0]);
|
||||
}
|
||||
|
||||
List<NonMaxSuppression.Box> boxes = new ArrayList<>(candidates.size());
|
||||
for (int i = 0; i < candidates.size(); i++) {
|
||||
RawDetection c = candidates.get(i);
|
||||
boxes.add(new NonMaxSuppression.Box(c.x1(), c.y1(), c.x2(), c.y2(), c.score(), i));
|
||||
}
|
||||
List<Integer> kept = NonMaxSuppression.suppress(boxes, NMS_THRESHOLD);
|
||||
|
||||
List<RawDetection> scaled = new ArrayList<>(kept.size());
|
||||
for (int index : kept) {
|
||||
RawDetection c = candidates.get(index);
|
||||
scaled.add(new RawDetection(c.x1() / letterbox.ratio(), c.y1() / letterbox.ratio(),
|
||||
c.x2() / letterbox.ratio(), c.y2() / letterbox.ratio(), c.score(), c.label()));
|
||||
}
|
||||
return scaled;
|
||||
}
|
||||
|
||||
/** {@code predictions}: {@code [numAnchors][85]} — 4 box + 1 objectness + 80 class scores, sigmoid already applied in-graph. */
|
||||
private static List<RawDetection> decode(float[][] predictions) {
|
||||
List<RawDetection> out = new ArrayList<>();
|
||||
int offset = 0;
|
||||
for (int stride : STRIDES) {
|
||||
int side = INPUT_SIZE / stride;
|
||||
for (int gy = 0; gy < side; gy++) {
|
||||
for (int gx = 0; gx < side; gx++) {
|
||||
float[] pred = predictions[offset + gy * side + gx];
|
||||
double cx = (pred[0] + gx) * stride;
|
||||
double cy = (pred[1] + gy) * stride;
|
||||
double w = Math.exp(pred[2]) * stride;
|
||||
double h = Math.exp(pred[3]) * stride;
|
||||
double obj = pred[4];
|
||||
|
||||
int bestClass = -1;
|
||||
double bestScore = 0;
|
||||
for (int c = ANIMAL_CLASS_MIN; c <= ANIMAL_CLASS_MAX; c++) {
|
||||
double classScore = pred[5 + c];
|
||||
if (classScore > bestScore) {
|
||||
bestScore = classScore;
|
||||
bestClass = c;
|
||||
}
|
||||
}
|
||||
double score = obj * bestScore;
|
||||
if (score < SCORE_THRESHOLD) {
|
||||
continue;
|
||||
}
|
||||
out.add(new RawDetection(cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2, score, COCO_CLASSES[bestClass]));
|
||||
}
|
||||
}
|
||||
offset += side * side;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private record Letterbox(BufferedImage image, double ratio) {
|
||||
}
|
||||
|
||||
/** Resizes preserving aspect ratio onto a {@code target}×{@code target} gray (114,114,114) canvas, content anchored top-left — YOLOX's own {@code preproc}. */
|
||||
private static Letterbox letterbox(BufferedImage source, int target) {
|
||||
double ratio = Math.min(target / (double) source.getHeight(), target / (double) source.getWidth());
|
||||
int newW = Math.round((float) (source.getWidth() * ratio));
|
||||
int newH = Math.round((float) (source.getHeight() * ratio));
|
||||
|
||||
BufferedImage canvas = new BufferedImage(target, target, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = canvas.createGraphics();
|
||||
try {
|
||||
g.setColor(new Color(114, 114, 114));
|
||||
g.fillRect(0, 0, target, target);
|
||||
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
|
||||
g.drawImage(source, 0, 0, newW, newH, null);
|
||||
}
|
||||
finally {
|
||||
g.dispose();
|
||||
}
|
||||
return new Letterbox(canvas, ratio);
|
||||
}
|
||||
|
||||
@Override
|
||||
@PreDestroy
|
||||
public void close() throws OrtException {
|
||||
session.close();
|
||||
}
|
||||
}
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import ai.onnxruntime.OnnxTensor;
|
||||
import ai.onnxruntime.OrtEnvironment;
|
||||
import ai.onnxruntime.OrtException;
|
||||
import ai.onnxruntime.OrtSession;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.icroco.pholio.domain.recognition.BoundingBox;
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.infra.recognition.engine.IFaceDetectionEngine;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.nio.FloatBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* {@link IFaceDetectionEngine} backed by two bundled ONNX models, chained: <a href="https://github.com/opencv/opencv_zoo/tree/main/models/face_detection_yunet">YuNet</a>
|
||||
* (bounding box + 5 landmarks) then <a href="https://github.com/opencv/opencv_zoo/tree/main/models/face_recognition_sface">SFace</a>
|
||||
* (128-d embedding, run on a landmark-aligned 112×112 crop) — both from OpenCV Zoo, MIT/Apache-2.0 licensed.
|
||||
* Every input tensor shape, output tensor name, decode formula and reference landmark constant below was
|
||||
* either taken verbatim from OpenCV's own {@code face_detect.cpp}/{@code face_recognize.cpp} (the C++ code
|
||||
* these ONNX graphs were designed to be driven by) or confirmed directly against the bundled {@code .onnx}
|
||||
* files via {@code OrtSession.getInputInfo()}/{@code getOutputInfo()} — not guessed.
|
||||
*
|
||||
* <p>{@code detectFaces} always runs both models: a face with no embedding would be useless to
|
||||
* {@code FaceClusteringService}, so there is no "detect only" mode to expose.
|
||||
*/
|
||||
@Component
|
||||
public class YuNetSFaceFaceDetectionEngine implements IFaceDetectionEngine, AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(YuNetSFaceFaceDetectionEngine.class);
|
||||
|
||||
private static final String SOURCE_PROVIDER = "local-onnx";
|
||||
|
||||
// --- YuNet: fixed 640x640 input (confirmed via OrtSession.getInputInfo()), 3 strides. ---
|
||||
private static final int YUNET_INPUT_SIZE = 640;
|
||||
private static final int[] YUNET_STRIDES = { 8, 16, 32 };
|
||||
/** opencv_zoo's own demo.py defaults — kept identical rather than re-tuned without reference photos to validate against. */
|
||||
private static final double YUNET_SCORE_THRESHOLD = 0.6;
|
||||
private static final double YUNET_NMS_THRESHOLD = 0.3;
|
||||
|
||||
// --- SFace: fixed 112x112 aligned input, 128-d output "fc1". ---
|
||||
private static final int SFACE_CROP_SIZE = 112;
|
||||
|
||||
/**
|
||||
* SFace's own reference layout (right eye, left eye, nose tip, right mouth corner, left mouth corner) in
|
||||
* 112×112 space — quoted verbatim from OpenCV's {@code face_recognize.cpp}. YuNet's 5 landmarks decode in
|
||||
* this exact same order, so no reordering is needed between the two models.
|
||||
*/
|
||||
private static final double[][] SFACE_REFERENCE_LANDMARKS = {
|
||||
{ 38.2946, 51.6963 }, { 73.5318, 51.5014 }, { 56.0252, 71.7366 }, { 41.5493, 92.3655 }, { 70.7299, 92.2041 }
|
||||
};
|
||||
|
||||
private final OrtEnvironment environment = OrtEnvironment.getEnvironment();
|
||||
private final OrtSession yunetSession;
|
||||
private final OrtSession sfaceSession;
|
||||
|
||||
public YuNetSFaceFaceDetectionEngine() {
|
||||
this.yunetSession = OnnxModelLoader.load("/models/recognition/face_detection_yunet_2023mar.onnx");
|
||||
this.sfaceSession = OnnxModelLoader.load("/models/recognition/face_recognition_sface_2021dec.onnx");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DetectedRegion> detectFaces(BufferedImage image) {
|
||||
try {
|
||||
List<RawFace> faces = detectRaw(image);
|
||||
List<DetectedRegion> regions = new ArrayList<>(faces.size());
|
||||
for (RawFace face : faces) {
|
||||
float[] embedding = embed(image, face.landmarks());
|
||||
double boxWidth = face.x2() - face.x1();
|
||||
double boxHeight = face.y2() - face.y1();
|
||||
BoundingBox box = new BoundingBox((face.x1() + boxWidth / 2) / image.getWidth(),
|
||||
(face.y1() + boxHeight / 2) / image.getHeight(),
|
||||
boxWidth / image.getWidth(), boxHeight / image.getHeight());
|
||||
regions.add(new DetectedRegion(EEntityKind.PERSON, box, face.score(), embedding, null, SOURCE_PROVIDER));
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
catch (OrtException e) {
|
||||
log.warn("Face detection/embedding failed unexpectedly: {}", e.toString(), e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private record RawFace(double x1, double y1, double x2, double y2, double score, double[] landmarks) {
|
||||
}
|
||||
|
||||
/** YuNet inference + per-stride decode + NMS, boxes/landmarks already scaled back to {@code image}'s own pixel space. */
|
||||
private List<RawFace> detectRaw(BufferedImage image) throws OrtException {
|
||||
BufferedImage squashed = resize(image, YUNET_INPUT_SIZE, YUNET_INPUT_SIZE);
|
||||
float[] chw = OnnxImageTensors.toChwFloats(squashed, true); // YuNet expects BGR (no swapRB in OpenCV's blobFromImage call)
|
||||
|
||||
List<RawFace> candidates = new ArrayList<>();
|
||||
try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, YUNET_INPUT_SIZE, YUNET_INPUT_SIZE });
|
||||
OrtSession.Result result = yunetSession.run(Map.of("input", input))) {
|
||||
for (int stride : YUNET_STRIDES) {
|
||||
decodeStride(result, stride, candidates);
|
||||
}
|
||||
}
|
||||
|
||||
List<NonMaxSuppression.Box> boxes = new ArrayList<>(candidates.size());
|
||||
for (int i = 0; i < candidates.size(); i++) {
|
||||
RawFace c = candidates.get(i);
|
||||
boxes.add(new NonMaxSuppression.Box(c.x1(), c.y1(), c.x2(), c.y2(), c.score(), i));
|
||||
}
|
||||
List<Integer> kept = NonMaxSuppression.suppress(boxes, YUNET_NMS_THRESHOLD);
|
||||
|
||||
double scaleX = image.getWidth() / (double) YUNET_INPUT_SIZE;
|
||||
double scaleY = image.getHeight() / (double) YUNET_INPUT_SIZE;
|
||||
List<RawFace> scaled = new ArrayList<>(kept.size());
|
||||
for (int index : kept) {
|
||||
RawFace c = candidates.get(index);
|
||||
double[] landmarks = new double[10];
|
||||
for (int i = 0; i < 5; i++) {
|
||||
landmarks[2 * i] = c.landmarks()[2 * i] * scaleX;
|
||||
landmarks[2 * i + 1] = c.landmarks()[2 * i + 1] * scaleY;
|
||||
}
|
||||
scaled.add(new RawFace(c.x1() * scaleX, c.y1() * scaleY, c.x2() * scaleX, c.y2() * scaleY, c.score(), landmarks));
|
||||
}
|
||||
return scaled;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void decodeStride(OrtSession.Result result, int stride, List<RawFace> out) throws OrtException {
|
||||
float[][][] cls = (float[][][]) result.get("cls_" + stride).orElseThrow().getValue();
|
||||
float[][][] obj = (float[][][]) result.get("obj_" + stride).orElseThrow().getValue();
|
||||
float[][][] bbox = (float[][][]) result.get("bbox_" + stride).orElseThrow().getValue();
|
||||
float[][][] kps = (float[][][]) result.get("kps_" + stride).orElseThrow().getValue();
|
||||
|
||||
int side = YUNET_INPUT_SIZE / stride;
|
||||
int count = cls[0].length;
|
||||
for (int idx = 0; idx < count; idx++) {
|
||||
int r = idx / side;
|
||||
int c = idx % side;
|
||||
double clsScore = clamp01(cls[0][idx][0]);
|
||||
double objScore = clamp01(obj[0][idx][0]);
|
||||
double score = Math.sqrt(clsScore * objScore);
|
||||
if (score < YUNET_SCORE_THRESHOLD) {
|
||||
continue;
|
||||
}
|
||||
float[] bb = bbox[0][idx];
|
||||
double cx = (c + bb[0]) * stride;
|
||||
double cy = (r + bb[1]) * stride;
|
||||
double w = Math.exp(bb[2]) * stride;
|
||||
double h = Math.exp(bb[3]) * stride;
|
||||
|
||||
float[] kp = kps[0][idx];
|
||||
double[] landmarks = new double[10];
|
||||
for (int n = 0; n < 5; n++) {
|
||||
landmarks[2 * n] = (kp[2 * n] + c) * stride;
|
||||
landmarks[2 * n + 1] = (kp[2 * n + 1] + r) * stride;
|
||||
}
|
||||
out.add(new RawFace(cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2, score, landmarks));
|
||||
}
|
||||
}
|
||||
|
||||
/** Aligns {@code image} onto SFace's reference layout using {@code landmarks} (5 points, original image coordinates), then embeds. */
|
||||
private float[] embed(BufferedImage image, double[] landmarks) throws OrtException {
|
||||
double[][] src = new double[5][2];
|
||||
for (int i = 0; i < 5; i++) {
|
||||
src[i][0] = landmarks[2 * i];
|
||||
src[i][1] = landmarks[2 * i + 1];
|
||||
}
|
||||
double[] t = SimilarityTransform.estimate(src, SFACE_REFERENCE_LANDMARKS);
|
||||
|
||||
BufferedImage aligned = new BufferedImage(SFACE_CROP_SIZE, SFACE_CROP_SIZE, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = aligned.createGraphics();
|
||||
try {
|
||||
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
|
||||
// dst = a*src + t, as an AffineTransform: x' = aRe*x - aIm*y + tRe ; y' = aIm*x + aRe*y + tIm
|
||||
g.drawImage(image, new AffineTransform(t[0], t[1], -t[1], t[0], t[2], t[3]), null);
|
||||
}
|
||||
finally {
|
||||
g.dispose();
|
||||
}
|
||||
|
||||
float[] chw = OnnxImageTensors.toChwFloats(aligned, false); // SFace swaps to RGB internally (swapRB=true)
|
||||
try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, SFACE_CROP_SIZE, SFACE_CROP_SIZE });
|
||||
OrtSession.Result result = sfaceSession.run(Map.of("data", input))) {
|
||||
float[][] embedding = (float[][]) result.get("fc1").orElseThrow().getValue();
|
||||
return l2Normalize(embedding[0]);
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] l2Normalize(float[] vector) {
|
||||
double norm = 0;
|
||||
for (float v : vector) {
|
||||
norm += v * v;
|
||||
}
|
||||
norm = Math.sqrt(norm);
|
||||
if (norm == 0) {
|
||||
return vector;
|
||||
}
|
||||
float[] normalized = new float[vector.length];
|
||||
for (int i = 0; i < vector.length; i++) {
|
||||
normalized[i] = (float) (vector[i] / norm);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static double clamp01(double value) {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
private static BufferedImage resize(BufferedImage source, int width, int height) {
|
||||
BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = resized.createGraphics();
|
||||
try {
|
||||
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
|
||||
g.drawImage(source, 0, 0, width, height, null);
|
||||
}
|
||||
finally {
|
||||
g.dispose();
|
||||
}
|
||||
return resized;
|
||||
}
|
||||
|
||||
@Override
|
||||
@PreDestroy
|
||||
public void close() throws OrtException {
|
||||
yunetSession.close();
|
||||
sfaceSession.close();
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import org.icroco.pholio.infra.library.MediaAnalysisService;
|
||||
import org.icroco.pholio.infra.library.MediaFileService;
|
||||
import org.icroco.pholio.infra.library.MediaMetadataEditService;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
|
||||
import org.icroco.pholio.infra.task.TaskService;
|
||||
import org.icroco.pholio.infra.task.TaskType;
|
||||
import org.icroco.pholio.ui.common.Disposable;
|
||||
@@ -139,7 +140,8 @@ public class GalleryView extends HBox implements Disposable, SelectionSource {
|
||||
MediaMetadataEditService metadataEditService,
|
||||
IPlaceSearchService placeSearchService,
|
||||
MediaAnalysisService mediaAnalysisService,
|
||||
GallerySearchState gallerySearchState) {
|
||||
GallerySearchState gallerySearchState,
|
||||
FaceRegionQueryService faceRegionQueryService) {
|
||||
this.libraryFolderService = libraryFolderService;
|
||||
this.preferences = preferences;
|
||||
this.fullImageCache = fullImageCache;
|
||||
@@ -155,7 +157,7 @@ public class GalleryView extends HBox implements Disposable, SelectionSource {
|
||||
galleryPane = new ThumbnailGalleryPane(mediaFileService, taskService, mediaLibraryState, preferences, i18n, imageCache);
|
||||
underConstructionPane = new UnderConstructionPane();
|
||||
detailPane = new PhotoDetailPane(fullImageCache);
|
||||
mediaInfoPane = new MediaInfoPane(i18n);
|
||||
mediaInfoPane = new MediaInfoPane(i18n, faceRegionQueryService);
|
||||
galleryPane.setOnOpenRequest((file, sourceThumbnail) -> openDetail(file, sourceThumbnail, null));
|
||||
galleryPane.setOnRegenerateThumbnails(this::regenerateThumbnails);
|
||||
galleryPane.setOnSetGpsForFiles(this::openLocationDialogForFiles);
|
||||
|
||||
@@ -17,6 +17,7 @@ import javafx.scene.control.Label;
|
||||
import javafx.scene.control.ScrollPane;
|
||||
import javafx.scene.input.Clipboard;
|
||||
import javafx.scene.input.ClipboardContent;
|
||||
import javafx.scene.layout.FlowPane;
|
||||
import javafx.scene.layout.HBox;
|
||||
import javafx.scene.layout.Priority;
|
||||
import javafx.scene.layout.StackPane;
|
||||
@@ -27,6 +28,8 @@ import javafx.util.Duration;
|
||||
import org.icroco.pholio.domain.library.MediaFile;
|
||||
import org.icroco.pholio.domain.media.GeoLocation;
|
||||
import org.icroco.pholio.domain.media.MediaMetadata;
|
||||
import org.icroco.pholio.domain.recognition.EEntityKind;
|
||||
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
|
||||
import org.icroco.pholio.ui.common.Disposable;
|
||||
import org.icroco.pholio.ui.control.StarRatingControl;
|
||||
import org.icroco.pholio.ui.i18n.I18nService;
|
||||
@@ -83,7 +86,8 @@ public class MediaInfoPane extends StackPane implements Disposable {
|
||||
/** How long {@link #setOpen} takes to grow/shrink this pane's own width. */
|
||||
private static final Duration SLIDE_DURATION = Duration.millis(240);
|
||||
|
||||
private final I18nService i18n;
|
||||
private final I18nService i18n;
|
||||
private final FaceRegionQueryService faceRegionQueryService;
|
||||
|
||||
private final Label title = new Label();
|
||||
private final Button closeButton = new Button();
|
||||
@@ -129,8 +133,9 @@ public class MediaInfoPane extends StackPane implements Disposable {
|
||||
private Consumer<MediaFile> onEditLocation = file -> {};
|
||||
private BiConsumer<MediaFile, Integer> onEditRating = (file, rating) -> {};
|
||||
|
||||
public MediaInfoPane(I18nService i18n) {
|
||||
public MediaInfoPane(I18nService i18n, FaceRegionQueryService faceRegionQueryService) {
|
||||
this.i18n = i18n;
|
||||
this.faceRegionQueryService = faceRegionQueryService;
|
||||
getStyleClass().add("media-info-pane");
|
||||
|
||||
// Starts fully collapsed — GalleryView seeds the real open/closed state right after construction.
|
||||
@@ -253,6 +258,7 @@ public class MediaInfoPane extends StackPane implements Disposable {
|
||||
nodes.add(ratingRow(file, metadata));
|
||||
nodes.add(dateRow(file, metadata));
|
||||
cameraRow(metadata).ifPresent(nodes::add);
|
||||
personsRow(file).ifPresent(nodes::add);
|
||||
nodes.add(fileRow(file, metadata));
|
||||
nodes.add(locationRow(file, metadata));
|
||||
Optional<GeoLocation> location = metadata.geoLocation();
|
||||
@@ -333,6 +339,35 @@ public class MediaInfoPane extends StackPane implements Disposable {
|
||||
return Optional.of(iconRow(Feather.CAMERA, camera.orElse(null), specs.isEmpty() ? null : String.join(" ", specs)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every person/animal {@code FaceRegionQueryService} has on file for {@code file}, one chip each — a
|
||||
* named person shows their name, an unnamed cluster (or the future confirmation panel not having run
|
||||
* yet) shows {@code "gallery.info.unknownPerson"}, and an animal shows its detected species. Read-only:
|
||||
* naming/confirming a region is the future person-management panel's job, not this pane's.
|
||||
*/
|
||||
private Optional<Node> personsRow(MediaFile file) {
|
||||
Long mediaFileId = file.id();
|
||||
if (mediaFileId == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
List<FaceRegionQueryService.DisplayRegion> regions = faceRegionQueryService.findDisplayRegionsFor(mediaFileId);
|
||||
if (regions.isEmpty()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
FlowPane chips = new FlowPane(6, 6);
|
||||
regions.forEach(region -> chips.getChildren().add(personChip(region)));
|
||||
return Optional.of(chips);
|
||||
}
|
||||
|
||||
private Label personChip(FaceRegionQueryService.DisplayRegion region) {
|
||||
String text = region.kind() == EEntityKind.PERSON
|
||||
? region.personName() != null ? region.personName() : i18n.get("gallery.info.unknownPerson")
|
||||
: region.label() != null ? region.label() : i18n.get("gallery.info.unknownPerson");
|
||||
Label chip = new Label(text);
|
||||
chip.getStyleClass().add("media-info-person-chip");
|
||||
return chip;
|
||||
}
|
||||
|
||||
private static String shutterLabel(double seconds) {
|
||||
return seconds >= 1
|
||||
? String.format(Locale.ROOT, "%.1fs", seconds)
|
||||
|
||||
@@ -371,6 +371,15 @@
|
||||
-fx-padding: 1 4 1 4;
|
||||
}
|
||||
|
||||
/* One detected person/animal on MediaInfoPane's persons row — a pill, same rounding convention as
|
||||
.media-info-edit-button, so an unnamed cluster ("Unknown person") reads as a real value, not a link. */
|
||||
.media-info-person-chip {
|
||||
-fx-background-color: -color-bg-inset;
|
||||
-fx-background-radius: 999px;
|
||||
-fx-padding: 3 10 3 10;
|
||||
-fx-font-size: 12px;
|
||||
}
|
||||
|
||||
/*
|
||||
* Transient outcome messages, top-right. The layer itself paints nothing: it is a click-through overlay,
|
||||
* and each toast is an AtlantaFX Notification carrying its own surface.
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
-- See V1's header comment: identifiers are quoted so H2 keeps them lower snake_case, matching what
|
||||
-- Spring Data JDBC generates.
|
||||
|
||||
-- One row per recognized identity. Only PERSON rows are linked to from media_face_region in this
|
||||
-- iteration — an ANIMAL region never gets one (species lives directly on media_face_region.label, no
|
||||
-- per-animal identity yet).
|
||||
CREATE TABLE "person"
|
||||
(
|
||||
"id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
"kind" VARCHAR(16) NOT NULL,
|
||||
"name" VARCHAR(255),
|
||||
"created_at" TIMESTAMP NOT NULL,
|
||||
CONSTRAINT "ck_person_kind" CHECK ("kind" IN ('PERSON', 'ANIMAL'))
|
||||
);
|
||||
|
||||
-- One row per detected face/animal bounding box. person_id stays NULL until FaceClusteringService (PERSON)
|
||||
-- links it; ANIMAL rows never get one. Re-detection replaces a file's rows wholesale (delete-then-reinsert),
|
||||
-- the same convention media_file_tag already uses for its own re-scans.
|
||||
CREATE TABLE "media_face_region"
|
||||
(
|
||||
"id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
"media_file_id" BIGINT NOT NULL REFERENCES "media_file" ("id") ON DELETE CASCADE,
|
||||
"person_id" BIGINT REFERENCES "person" ("id") ON DELETE SET NULL,
|
||||
"kind" VARCHAR(16) NOT NULL,
|
||||
-- Normalized MWG-RS "stArea": x/y is the region's CENTER, w/h its size, fractions of the full image
|
||||
-- (0..1) — identical numbers to what an mwg-rs:Area XMP struct stores, zero conversion on round-trip.
|
||||
"area_x" DOUBLE NOT NULL,
|
||||
"area_y" DOUBLE NOT NULL,
|
||||
"area_w" DOUBLE NOT NULL,
|
||||
"area_h" DOUBLE NOT NULL,
|
||||
"confidence" DOUBLE NOT NULL,
|
||||
"embedding" VARBINARY(8192),
|
||||
"label" VARCHAR(255),
|
||||
"source_provider" VARCHAR(255) NOT NULL,
|
||||
"confirmed" BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
"detected_at" TIMESTAMP NOT NULL,
|
||||
CONSTRAINT "ck_media_face_region_kind" CHECK ("kind" IN ('PERSON', 'ANIMAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX "ix_media_face_region_media_file" ON "media_face_region" ("media_file_id");
|
||||
CREATE INDEX "ix_media_face_region_person" ON "media_face_region" ("person_id");
|
||||
@@ -102,6 +102,7 @@ gallery.unknownDate=Unknown date
|
||||
gallery.info.hash=Hash
|
||||
gallery.info.noMetadata=No metadata available
|
||||
gallery.info.addPlace=Add a location
|
||||
gallery.info.unknownPerson=Unknown person
|
||||
gallery.location.edit.title=Add a location
|
||||
gallery.location.edit.searchPrompt=Search for a place…
|
||||
gallery.location.edit.hint=Changes to the place a photo was taken are saved to the library and, when the format supports it, to the file itself.
|
||||
@@ -170,6 +171,10 @@ settings.geocoding.cities5000LastImport=Cities5000 reference data last imported
|
||||
settings.geocoding.cities5000RowCount=Cities5000 reference row count
|
||||
settings.geocoding.providersJson=Address search providers
|
||||
settings.geocoding.activeProvider=Active address search provider
|
||||
settings.recognition.enabled=Face/animal recognition enabled
|
||||
settings.recognition.providersJson=Recognition providers
|
||||
settings.recognition.activeProvider=Active recognition provider
|
||||
settings.recognition.clusterThreshold=Person clustering similarity threshold
|
||||
settings.ai.provider=Provider
|
||||
settings.ai.endpoint=Endpoint
|
||||
settings.imports.largeFolderThreshold=Confirm above (files)
|
||||
|
||||
@@ -104,6 +104,7 @@ gallery.unknownDate=Date inconnue
|
||||
gallery.info.hash=Hash
|
||||
gallery.info.noMetadata=Aucune métadonnée disponible
|
||||
gallery.info.addPlace=Ajouter un lieu
|
||||
gallery.info.unknownPerson=Personne inconnue
|
||||
gallery.location.edit.title=Ajouter un lieu
|
||||
gallery.location.edit.searchPrompt=Rechercher un lieu…
|
||||
gallery.location.edit.hint=Les modifications apportées au lieu de prise de vue seront enregistrées dans la photothèque et, si le format le permet, dans le fichier lui-même.
|
||||
@@ -172,6 +173,10 @@ settings.geocoding.cities5000LastImport=Dernière importation des données de r
|
||||
settings.geocoding.cities5000RowCount=Nombre de lignes de référence Cities5000
|
||||
settings.geocoding.providersJson=Fournisseurs de recherche d'adresse
|
||||
settings.geocoding.activeProvider=Fournisseur de recherche d'adresse actif
|
||||
settings.recognition.enabled=Reconnaissance de visages/animaux activée
|
||||
settings.recognition.providersJson=Fournisseurs de reconnaissance
|
||||
settings.recognition.activeProvider=Fournisseur de reconnaissance actif
|
||||
settings.recognition.clusterThreshold=Seuil de similarité pour le regroupement de personnes
|
||||
settings.ai.provider=Fournisseur
|
||||
settings.ai.endpoint=Point d'accès
|
||||
settings.imports.largeFolderThreshold=Confirmer au-delà de (fichiers)
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# Bundled recognition models
|
||||
|
||||
Third-party ONNX weights bundled with Pholio for local face/animal recognition. None were modified —
|
||||
each is used exactly as published upstream.
|
||||
|
||||
## face_detection_yunet_2023mar.onnx
|
||||
|
||||
- Source: [opencv/opencv_zoo](https://github.com/opencv/opencv_zoo/tree/main/models/face_detection_yunet)
|
||||
- License: MIT
|
||||
- Used by: `YuNetSFaceFaceDetectionEngine` (face bounding box + 5-point landmarks)
|
||||
|
||||
## face_recognition_sface_2021dec.onnx
|
||||
|
||||
- Source: [opencv/opencv_zoo](https://github.com/opencv/opencv_zoo/tree/main/models/face_recognition_sface)
|
||||
- License: Apache-2.0
|
||||
- Used by: `YuNetSFaceFaceDetectionEngine` (128-d face embedding, for `FaceClusteringService`)
|
||||
|
||||
## yolox_nano.onnx
|
||||
|
||||
- Source: [Megvii-BaseDetection/YOLOX](https://github.com/Megvii-BaseDetection/YOLOX), release `0.1.1rc0`
|
||||
- License: Apache-2.0
|
||||
- Used by: `YoloXAnimalDetectionEngine` (COCO 80-class detector, filtered to animal classes 14-23)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -281,6 +281,42 @@ geocoding:
|
||||
visible: false
|
||||
editable: false
|
||||
|
||||
# Local/remote face+animal recognition — same shape as `geocoding` above, edited only through Maintenance's
|
||||
# recognition tab, never through this generic settings form.
|
||||
recognition:
|
||||
enabled:
|
||||
type: BOOLEAN
|
||||
label: settings.recognition.enabled
|
||||
default-value: true
|
||||
visible: false
|
||||
editable: false
|
||||
|
||||
# Edited only through Maintenance's recognition tab — a JSON-encoded list of RecognitionProviderConfig.
|
||||
providers-json:
|
||||
type: STRING
|
||||
label: settings.recognition.providersJson
|
||||
default-value: "[]"
|
||||
visible: false
|
||||
editable: false
|
||||
|
||||
# The active RecognitionProviderConfig's name, or blank to mean "local only" — see RecognitionService.
|
||||
active-provider:
|
||||
type: STRING
|
||||
label: settings.recognition.activeProvider
|
||||
default-value: ""
|
||||
visible: false
|
||||
editable: false
|
||||
|
||||
# Cosine-similarity threshold above which FaceClusteringService links an unnamed face to an existing
|
||||
# Person cluster rather than minting a new one. 0.363 is SFace's own calibrated same-identity threshold
|
||||
# (OpenCV Zoo's face_recognition_sface README) — the embedding is SFace's, so its own threshold applies.
|
||||
person-cluster-threshold:
|
||||
type: DOUBLE
|
||||
label: settings.recognition.clusterThreshold
|
||||
default-value: 0.363
|
||||
visible: false
|
||||
editable: false
|
||||
|
||||
# Onboarding, shown once on the first run. Not offered in the settings view: there is nothing to configure,
|
||||
# only a fact to remember once the coach-mark sequence has been dismissed or completed.
|
||||
onboarding:
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class NonMaxSuppressionTest {
|
||||
|
||||
@Test
|
||||
void suppressesTheLowerScoredOfTwoHeavilyOverlappingBoxes() {
|
||||
List<NonMaxSuppression.Box> boxes = List.of(
|
||||
new NonMaxSuppression.Box(0, 0, 10, 10, 0.9, 0),
|
||||
new NonMaxSuppression.Box(1, 1, 11, 11, 0.5, 1));
|
||||
|
||||
List<Integer> kept = NonMaxSuppression.suppress(boxes, 0.3);
|
||||
|
||||
assertThat(kept).containsExactly(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void keepsBothOfTwoNonOverlappingBoxes() {
|
||||
List<NonMaxSuppression.Box> boxes = List.of(
|
||||
new NonMaxSuppression.Box(0, 0, 10, 10, 0.9, 0),
|
||||
new NonMaxSuppression.Box(100, 100, 110, 110, 0.5, 1));
|
||||
|
||||
List<Integer> kept = NonMaxSuppression.suppress(boxes, 0.3);
|
||||
|
||||
assertThat(kept).containsExactlyInAnyOrder(0, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void ordersSurvivorsHighestScoreFirst() {
|
||||
List<NonMaxSuppression.Box> boxes = List.of(
|
||||
new NonMaxSuppression.Box(0, 0, 10, 10, 0.4, 0),
|
||||
new NonMaxSuppression.Box(100, 100, 110, 110, 0.9, 1));
|
||||
|
||||
List<Integer> kept = NonMaxSuppression.suppress(boxes, 0.3);
|
||||
|
||||
assertThat(kept).containsExactly(1, 0);
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import org.assertj.core.api.SoftAssertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.within;
|
||||
|
||||
class SimilarityTransformTest {
|
||||
|
||||
@Test
|
||||
void recoversAnExactUniformScaleAndTranslation() {
|
||||
double[][] src = { { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } };
|
||||
double[][] dst = new double[src.length][2];
|
||||
for (int i = 0; i < src.length; i++) {
|
||||
dst[i][0] = 2 * src[i][0] + 10;
|
||||
dst[i][1] = 2 * src[i][1] + 5;
|
||||
}
|
||||
|
||||
double[] t = SimilarityTransform.estimate(src, dst);
|
||||
|
||||
SoftAssertions.assertSoftly(softly -> {
|
||||
softly.assertThat(t[0]).as("scale (real part)").isCloseTo(2.0, within(1e-9));
|
||||
softly.assertThat(t[1]).as("rotation (imaginary part)").isCloseTo(0.0, within(1e-9));
|
||||
softly.assertThat(t[2]).as("translation x").isCloseTo(10.0, within(1e-9));
|
||||
softly.assertThat(t[3]).as("translation y").isCloseTo(5.0, within(1e-9));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void recoversAnExactRotation() {
|
||||
// 90 degrees counter-clockwise about the origin: (x, y) -> (-y, x)
|
||||
double[][] src = { { 1, 0 }, { 0, 1 }, { -1, 0 }, { 0, -1 } };
|
||||
double[][] dst = { { 0, 1 }, { -1, 0 }, { 0, -1 }, { 1, 0 } };
|
||||
|
||||
double[] t = SimilarityTransform.estimate(src, dst);
|
||||
|
||||
SoftAssertions.assertSoftly(softly -> {
|
||||
softly.assertThat(t[0]).as("scale (real part)").isCloseTo(0.0, within(1e-9));
|
||||
softly.assertThat(t[1]).as("rotation (imaginary part)").isCloseTo(1.0, within(1e-9));
|
||||
softly.assertThat(t[2]).as("translation x").isCloseTo(0.0, within(1e-9));
|
||||
softly.assertThat(t[3]).as("translation y").isCloseTo(0.0, within(1e-9));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void identityWhenSourceAlreadyMatchesDestination() {
|
||||
double[][] points = { { 3, 4 }, { 5, 1 }, { -2, 7 } };
|
||||
|
||||
double[] t = SimilarityTransform.estimate(points, points);
|
||||
|
||||
SoftAssertions.assertSoftly(softly -> {
|
||||
softly.assertThat(t[0]).isCloseTo(1.0, within(1e-9));
|
||||
softly.assertThat(t[1]).isCloseTo(0.0, within(1e-9));
|
||||
softly.assertThat(t[2]).isCloseTo(0.0, within(1e-9));
|
||||
softly.assertThat(t[3]).isCloseTo(0.0, within(1e-9));
|
||||
});
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import ai.onnxruntime.OrtException;
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** Smoke test — see {@link YuNetSFaceFaceDetectionEngineTest}'s own javadoc for why this isn't an accuracy test. */
|
||||
class YoloXAnimalDetectionEngineTest {
|
||||
|
||||
@Test
|
||||
void loadsBundledModelAndRunsInferenceWithoutThrowing() throws OrtException {
|
||||
try (YoloXAnimalDetectionEngine engine = new YoloXAnimalDetectionEngine()) {
|
||||
BufferedImage image = randomImage(640, 480);
|
||||
|
||||
List<DetectedRegion> regions = engine.detectAnimals(image);
|
||||
|
||||
assertThat(regions).isNotNull();
|
||||
}
|
||||
}
|
||||
|
||||
private static BufferedImage randomImage(int width, int height) {
|
||||
BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
Random random = new Random(42);
|
||||
for (int y = 0; y < height; y++) {
|
||||
for (int x = 0; x < width; x++) {
|
||||
image.setRGB(x, y, random.nextInt(0xFFFFFF));
|
||||
}
|
||||
}
|
||||
return image;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.icroco.pholio.infra.recognition.engine.onnx;
|
||||
|
||||
import ai.onnxruntime.OrtException;
|
||||
import org.icroco.pholio.domain.recognition.DetectedRegion;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* A smoke test, not an accuracy test: no real face photo ships with this repo (nothing to license/attribute,
|
||||
* nothing that could be mistaken for real personal data), so this only pins down that the bundled ONNX
|
||||
* models load and run inference without throwing on this exact JDK/ONNX Runtime combination — the actual
|
||||
* risk flagged in the recognition feature's implementation plan. A synthetic noise image legitimately finds
|
||||
* zero faces; that is the correct, expected answer, not a test gap.
|
||||
*/
|
||||
class YuNetSFaceFaceDetectionEngineTest {
|
||||
|
||||
@Test
|
||||
void loadsBundledModelsAndRunsInferenceWithoutThrowing() throws OrtException {
|
||||
try (YuNetSFaceFaceDetectionEngine engine = new YuNetSFaceFaceDetectionEngine()) {
|
||||
BufferedImage image = randomImage(640, 480);
|
||||
|
||||
List<DetectedRegion> regions = engine.detectFaces(image);
|
||||
|
||||
assertThat(regions).isNotNull();
|
||||
}
|
||||
}
|
||||
|
||||
private static BufferedImage randomImage(int width, int height) {
|
||||
BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
Random random = new Random(42);
|
||||
for (int y = 0; y < height; y++) {
|
||||
for (int x = 0; x < width; x++) {
|
||||
image.setRGB(x, y, random.nextInt(0xFFFFFF));
|
||||
}
|
||||
}
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -62,7 +62,7 @@ class MediaLibraryStateTest {
|
||||
|
||||
boolean readyBefore = state.thumbnailReadyProperty(1L).get();
|
||||
boolean otherFileReady = state.thumbnailReadyProperty(2L).get();
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
|
||||
awaitFlush();
|
||||
|
||||
assertThat(readyBefore).isFalse();
|
||||
@@ -97,7 +97,7 @@ class MediaLibraryStateTest {
|
||||
void folderRemovedClearsEveryThumbnailReadyProperty() throws InterruptedException {
|
||||
FxTestToolkit.requireToolkit();
|
||||
MediaLibraryState state = new MediaLibraryState();
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
|
||||
awaitFlush();
|
||||
assertThat(state.thumbnailReadyProperty(1L).get()).isTrue();
|
||||
|
||||
@@ -136,7 +136,7 @@ class MediaLibraryStateTest {
|
||||
void libraryChangedClearsEveryThumbnailReadyProperty() throws InterruptedException {
|
||||
FxTestToolkit.requireToolkit();
|
||||
MediaLibraryState state = new MediaLibraryState();
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
|
||||
awaitFlush();
|
||||
assertThat(state.thumbnailReadyProperty(1L).get()).isTrue();
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import org.icroco.pholio.infra.library.MediaFileService;
|
||||
import org.icroco.pholio.infra.library.MediaMetadataEditService;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.icroco.pholio.infra.preferences.PreferencesFixture;
|
||||
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
|
||||
import org.icroco.pholio.infra.task.TaskService;
|
||||
import org.icroco.pholio.ui.FxTestToolkit;
|
||||
import org.icroco.pholio.ui.ViewSwitcher;
|
||||
@@ -345,7 +346,8 @@ class NavigationRailTest {
|
||||
mock(MediaMetadataEditService.class),
|
||||
mock(IPlaceSearchService.class),
|
||||
mock(MediaAnalysisService.class),
|
||||
new GallerySearchState());
|
||||
new GallerySearchState(),
|
||||
mock(FaceRegionQueryService.class));
|
||||
when(context.getBean(GalleryView.class)).thenReturn(galleryView);
|
||||
when(context.getBean(ModulePlaceholderView.class)).thenReturn(new ModulePlaceholderView(i18n));
|
||||
return new ViewSwitcher(context, new ViewportSelection());
|
||||
|
||||
@@ -22,6 +22,7 @@ import org.icroco.pholio.infra.library.MediaFileService;
|
||||
import org.icroco.pholio.infra.library.MediaMetadataEditService;
|
||||
import org.icroco.pholio.infra.preferences.AppPreferences;
|
||||
import org.icroco.pholio.infra.preferences.PreferencesFixture;
|
||||
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
|
||||
import org.icroco.pholio.infra.task.TaskService;
|
||||
import org.icroco.pholio.ui.FxTestToolkit;
|
||||
import org.icroco.pholio.ui.i18n.I18nService;
|
||||
@@ -87,7 +88,8 @@ class GalleryViewTest {
|
||||
mock(MediaMetadataEditService.class),
|
||||
mock(IPlaceSearchService.class),
|
||||
mock(MediaAnalysisService.class),
|
||||
new GallerySearchState()));
|
||||
new GallerySearchState(),
|
||||
mock(FaceRegionQueryService.class)));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -214,7 +216,7 @@ class GalleryViewTest {
|
||||
new ThumbnailImageCache(localPreferences, taskService),
|
||||
new FullImageCache(localPreferences, taskService), mock(LibraryFolderService.class),
|
||||
mock(ModalService.class), mock(MediaMetadataEditService.class), mock(IPlaceSearchService.class),
|
||||
mock(MediaAnalysisService.class), new GallerySearchState()));
|
||||
mock(MediaAnalysisService.class), new GallerySearchState(), mock(FaceRegionQueryService.class)));
|
||||
|
||||
Stage farStage = onFxThread(() -> {
|
||||
farView.resize(400, 500);
|
||||
|
||||
@@ -518,7 +518,7 @@ class ThumbnailGalleryPaneTest {
|
||||
clearInvocations(mediaFileService);
|
||||
List<GalleryRow> rowsBefore = pane.rows().getItems();
|
||||
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
|
||||
awaitFlush();
|
||||
|
||||
assertThat(pane.rows().getItems()).as("same instance: no relayout, only the one cell's slot changed")
|
||||
@@ -565,7 +565,7 @@ class ThumbnailGalleryPaneTest {
|
||||
Files.createDirectories(thumbnail.getParent());
|
||||
Files.write(thumbnail, new byte[]{1, 2, 3});
|
||||
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
|
||||
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
|
||||
awaitFlush();
|
||||
|
||||
Node cardAfter = onFxThread(() -> {
|
||||
|
||||
Reference in New Issue
Block a user