File 4052_support_focus_point.patch of Package gotosocial
From 0d27ed18d1c6fe5c6e9f9a677193f5519dfb87a3 Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Sat, 26 Apr 2025 15:03:05 +0200
Subject: [PATCH] [feature] Update attachment format, receive + send
`focalPoint` prop + use it on the frontend (#4052)
* [feature] Update attachment format, receive + send `focalPoint` prop + use it on the frontend
* whoops
* boop
* restore function signature of ExtractAttachments
---
docs/federation/posts.md | 51 +++++++
internal/ap/extract.go | 65 ++++++++-
internal/ap/extractfocus_test.go | 125 +++++++++++++++++
internal/ap/interfaces.go | 31 ++++-
internal/ap/properties.go | 64 +++++++++
.../api/client/statuses/statusboost_test.go | 4 +-
internal/federation/dereferencing/status.go | 2 +
internal/filter/spam/statusable.go | 9 +-
internal/router/template.go | 10 ++
internal/typeutils/internaltoas.go | 126 ++++++++++++------
internal/typeutils/internaltoas_test.go | 24 +++-
internal/typeutils/internaltofrontend_test.go | 24 ++--
testrig/testmodels.go | 6 +
web/source/frontend/index.js | 5 +
.../frontend/photoswipe-object-position.js | 119 +++++++++++++++++
web/source/frontend_prerender/index.js | 12 ++
web/source/index.js | 2 +-
web/template/status_attachment.tmpl | 8 ++
18 files changed, 616 insertions(+), 71 deletions(-)
create mode 100644 internal/ap/extractfocus_test.go
create mode 100644 web/source/frontend/photoswipe-object-position.js
diff --git a/docs/federation/posts.md b/docs/federation/posts.md
index 63248f058..45a4fb7cd 100644
--- a/docs/federation/posts.md
+++ b/docs/federation/posts.md
@@ -1,5 +1,56 @@
# Posts and Post Properties
+## Attachments, Blurhash, and Focal Point
+
+GoToSocial sends media attachments in the `attachment` property of posts using the following types:
+
+- `Image` - any image type (webp, jpeg, gif, png, etc).
+- `Video` - any video type (mp4, mkv, webm, etc).
+- `Audio` - any audio type (mp3, flac, wma, etc).
+- `Document` - fallback for any other / unknown type.
+
+Attachments sent from GoToSocial include the MIME `mediaType`, the `url` of the full-sized version of the media file, and the `summary` property, which can be interpreted by remote instance's as a short description / alt text for the attachment.
+
+Types `Image` and `Video` will also include the `http://joinmastodon.org/ns#blurhash` property so that remotes can generate a colorful hash of the image. If an audio file included an embedded cover art image, then the `Audio` type will also include a blurhash. See the [Mastodon blurhash docs](https://docs.joinmastodon.org/spec/activitypub/#blurhash).
+
+`Image` types may also include the `http://joinmastodon.org/ns#focalPoint` property, which will be an array of two floats between -1.0 and 1.0 indicating the x-y coordinates of the image's focal point. See the [Mastondon focalPoint docs](https://docs.joinmastodon.org/spec/activitypub/#focalPoint).
+
+Here's an example of a `Note` with one attachment:
+
+```json
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ {
+ "blurhash": "toot:blurhash",
+ "focalPoint": {
+ "@container": "@list",
+ "@id": "toot:focalPoint"
+ },
+ "toot": "http://joinmastodon.org/ns#"
+ }
+ ],
+ "type": "Note",
+ [...],
+ "attachment": [
+ {
+ "blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
+ "focalPoint": [
+ -0.5,
+ 0.5
+ ],
+ "mediaType": "image/jpeg",
+ "summary": "Black and white image of some 50's style text saying: Welcome On Board",
+ "type": "Image",
+ "url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
+ }
+ ],
+ [...]
+}
+```
+
+When receiving posts with attachments from remote instances, it will try to parse any of the four types `Image`, `Video`, `Audio`, or `Document` into media attachments. It doesn't matter which type is used. It will check for `blurhash` and `focalPoint` properties and use these if they are set. It will use the `summary` value as a short description / alt text, falling back to `name` if `summary` is not set.
+
## Hashtags
GoToSocial users can include hashtags in their posts, which indicate to other instances that that user wishes their post to be grouped together with other posts using the same hashtag, for discovery purposes.
diff --git a/internal/ap/extract.go b/internal/ap/extract.go
index 2387c1f9a..40578b8fc 100644
--- a/internal/ap/extract.go
+++ b/internal/ap/extract.go
@@ -634,32 +634,38 @@ func ExtractContent(i WithContent) gtsmodel.Content {
return content
}
-// ExtractAttachments attempts to extract barebones MediaAttachment objects from given AS interface type.
+// ExtractAttachments attempts to extract barebones
+// MediaAttachment objects from given AS interface type.
func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) {
attachmentProp := i.GetActivityStreamsAttachment()
if attachmentProp == nil {
return nil, nil
}
- var errs gtserror.MultiError
+ var (
+ attachments = make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
+ errs gtserror.MultiError
+ )
- attachments := make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
t := iter.GetType()
if t == nil {
errs.Appendf("nil attachment type")
continue
}
- attachmentable, ok := t.(Attachmentable)
+
+ attachmentable, ok := ToAttachmentable(t)
if !ok {
- errs.Appendf("incorrect attachment type: %T", t)
+ errs.Appendf("could not cast %T to Attachmentable", t)
continue
}
+
attachment, err := ExtractAttachment(attachmentable)
if err != nil {
errs.Appendf("error extracting attachment: %w", err)
continue
}
+
attachments = append(attachments, attachment)
}
@@ -681,7 +687,10 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
RemoteURL: remoteURL.String(),
Description: ExtractDescription(i),
Blurhash: ExtractBlurhash(i),
- Processing: gtsmodel.ProcessingStatusReceived,
+ FileMeta: gtsmodel.FileMeta{
+ Focus: ExtractFocus(i),
+ },
+ Processing: gtsmodel.ProcessingStatusReceived,
}, nil
}
@@ -708,6 +717,50 @@ func ExtractBlurhash(i WithBlurhash) string {
return blurhashProp.Get()
}
+// ExtractFocus parses a gtsmodel.Focus from the given Attachmentable's
+// `focalPoint` property, if Attachmentable can have `focalPoint`, and
+// `focalPoint` is set to a valid pair of floats. Otherwise, returns a
+// zero gtsmodel.Focus (ie., focus in the centre of the image).
+func ExtractFocus(attachmentable Attachmentable) gtsmodel.Focus {
+ focus := gtsmodel.Focus{}
+
+ withFocalPoint, ok := attachmentable.(WithFocalPoint)
+ if !ok {
+ return focus
+ }
+
+ focalPointProp := withFocalPoint.GetTootFocalPoint()
+ if focalPointProp == nil || focalPointProp.Len() != 2 {
+ return focus
+ }
+
+ xProp := focalPointProp.At(0)
+ if !xProp.IsXMLSchemaFloat() {
+ return focus
+ }
+
+ yProp := focalPointProp.At(1)
+ if !yProp.IsXMLSchemaFloat() {
+ return focus
+ }
+
+ x := xProp.Get()
+ if x < -1 || x > 1 {
+ return focus
+ }
+
+ y := yProp.Get()
+ if y < -1 || y > 1 {
+ return focus
+ }
+
+ // Looks good.
+ focus.X = float32(x)
+ focus.Y = float32(y)
+
+ return focus
+}
+
// ExtractHashtags extracts a slice of minimal gtsmodel.Tags
// from a WithTag. If an entry in the WithTag is not a hashtag,
// or has a name that cannot be normalized, it will be ignored.
diff --git a/internal/ap/extractfocus_test.go b/internal/ap/extractfocus_test.go
new file mode 100644
index 000000000..9e7935740
--- /dev/null
+++ b/internal/ap/extractfocus_test.go
@@ -0,0 +1,125 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package ap_test
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "testing"
+
+ "code.superseriousbusiness.org/activity/streams"
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/ap"
+)
+
+type ExtractFocusTestSuite struct {
+ APTestSuite
+}
+
+func (suite *ExtractFocusTestSuite) TestExtractFocus() {
+ ctx := context.Background()
+
+ type test struct {
+ data string
+ expectX float32
+ expectY float32
+ }
+
+ for _, test := range []test{
+ {
+ // Fine.
+ data: "-0.5, 0.5",
+ expectX: -0.5,
+ expectY: 0.5,
+ },
+ {
+ // Also fine.
+ data: "1, 1",
+ expectX: 1,
+ expectY: 1,
+ },
+ {
+ // Out of range.
+ data: "1.5, 1",
+ expectX: 0,
+ expectY: 0,
+ },
+ {
+ // Too many points.
+ data: "1, 1, 0",
+ expectX: 0,
+ expectY: 0,
+ },
+ {
+ // Not enough points.
+ data: "1",
+ expectX: 0,
+ expectY: 0,
+ },
+ } {
+ // Wrap provided test.data
+ // in a minimal Attachmentable.
+ const fmts = `{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ {
+ "focalPoint": {
+ "@container": "@list",
+ "@id": "toot:focalPoint"
+ },
+ "toot": "http://joinmastodon.org/ns#"
+ }
+ ],
+ "focalPoint": [ %s ],
+ "type": "Image"
+}`
+
+ // Unmarshal test data.
+ data := fmt.Sprintf(fmts, test.data)
+ m := make(map[string]any)
+ if err := json.Unmarshal([]byte(data), &m); err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // Convert to type.
+ t, err := streams.ToType(ctx, m)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ // Convert to attachmentable.
+ attachmentable, ok := t.(ap.Attachmentable)
+ if !ok {
+ suite.FailNow("", "%T was not Attachmentable", t)
+ }
+
+ // Check extracted focus.
+ focus := ap.ExtractFocus(attachmentable)
+ if focus.X != test.expectX || focus.Y != test.expectY {
+ suite.Fail("",
+ "expected x=%.2f y=%.2f got x=%.2f y=%.2f",
+ test.expectX, test.expectY, focus.X, focus.Y,
+ )
+ }
+ }
+}
+
+func TestExtractFocusTestSuite(t *testing.T) {
+ suite.Run(t, new(ExtractFocusTestSuite))
+}
diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go
index 1dcc6afef..28b5c0d20 100644
--- a/internal/ap/interfaces.go
+++ b/internal/ap/interfaces.go
@@ -165,6 +165,29 @@ func ToApprovable(t vocab.Type) (Approvable, bool) {
return approvable, true
}
+// IsAttachmentable returns whether AS vocab type name
+// is something that can be cast to Attachmentable.
+func IsAttachmentable(typeName string) bool {
+ switch typeName {
+ case ObjectAudio,
+ ObjectDocument,
+ ObjectImage,
+ ObjectVideo:
+ return true
+ default:
+ return false
+ }
+}
+
+// ToAttachmentable safely tries to cast vocab.Type as Attachmentable.
+func ToAttachmentable(t vocab.Type) (Attachmentable, bool) {
+ attachmentable, ok := t.(Attachmentable)
+ if !ok || !IsAttachmentable(t.GetTypeName()) {
+ return nil, false
+ }
+ return attachmentable, true
+}
+
// Activityable represents the minimum activitypub interface for representing an 'activity'.
// (see: IsActivityable() for types implementing this, though you MUST make sure to check
// the typeName as this bare interface may be implementable by non-Activityable types).
@@ -628,9 +651,11 @@ type WithBlurhash interface {
SetTootBlurhash(vocab.TootBlurhashProperty)
}
-// type withFocalPoint interface {
-// // TODO
-// }
+// WithFocalPoint represents an object with TootFocalPointProperty.
+type WithFocalPoint interface {
+ GetTootFocalPoint() vocab.TootFocalPointProperty
+ SetTootFocalPoint(vocab.TootFocalPointProperty)
+}
// WithHref represents an activity with ActivityStreamsHrefProperty
type WithHref interface {
diff --git a/internal/ap/properties.go b/internal/ap/properties.go
index d1ef62972..b5154c0aa 100644
--- a/internal/ap/properties.go
+++ b/internal/ap/properties.go
@@ -560,6 +560,70 @@ func SetApprovedBy(with WithApprovedBy, approvedBy *url.URL) {
abProp.Set(approvedBy)
}
+// GetMediaType returns the string contained in
+// the MediaType property of 'with', if set.
+func GetMediaType(with WithMediaType) string {
+ mtProp := with.GetActivityStreamsMediaType()
+ if mtProp == nil || !mtProp.IsRFCRfc2045() {
+ return ""
+ }
+ return mtProp.Get()
+}
+
+// SetMediaType sets the given string
+// on the MediaType property of 'with'.
+func SetMediaType(with WithMediaType, mediaType string) {
+ mtProp := with.GetActivityStreamsMediaType()
+ if mtProp == nil {
+ mtProp = streams.NewActivityStreamsMediaTypeProperty()
+ with.SetActivityStreamsMediaType(mtProp)
+ }
+ mtProp.Set(mediaType)
+}
+
+// AppendName appends the given name
+// vals to the Name property of 'with'.
+func AppendName(with WithName, name ...string) {
+ if len(name) == 0 {
+ return
+ }
+ nameProp := with.GetActivityStreamsName()
+ if nameProp == nil {
+ nameProp = streams.NewActivityStreamsNameProperty()
+ with.SetActivityStreamsName(nameProp)
+ }
+ for _, name := range name {
+ nameProp.AppendXMLSchemaString(name)
+ }
+}
+
+// AppendSummary appends the given summary
+// vals to the Summary property of 'with'.
+func AppendSummary(with WithSummary, summary ...string) {
+ if len(summary) == 0 {
+ return
+ }
+ summaryProp := with.GetActivityStreamsSummary()
+ if summaryProp == nil {
+ summaryProp = streams.NewActivityStreamsSummaryProperty()
+ with.SetActivityStreamsSummary(summaryProp)
+ }
+ for _, summary := range summary {
+ summaryProp.AppendXMLSchemaString(summary)
+ }
+}
+
+// SetBlurhash sets the given string
+// on the Blurhash property of 'with'.
+func SetBlurhash(with WithBlurhash, mediaType string) {
+ bProp := with.GetTootBlurhash()
+ if bProp == nil {
+ bProp = streams.NewTootBlurhashProperty()
+ with.SetTootBlurhash(bProp)
+ }
+ bProp.Set(mediaType)
+}
+
// extractIRIs extracts just the AP IRIs from an iterable
// property that may contain types (with IRIs) or just IRIs.
//
diff --git a/internal/api/client/statuses/statusboost_test.go b/internal/api/client/statuses/statusboost_test.go
index 4c8d41a99..333334bbc 100644
--- a/internal/api/client/statuses/statusboost_test.go
+++ b/internal/api/client/statuses/statusboost_test.go
@@ -193,8 +193,8 @@ func (suite *StatusBoostTestSuite) TestPostBoost() {
"id": "01F8MH6NEM8D7527KZAECTCR76",
"meta": {
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
},
"original": {
"aspect": 1.9047619,
diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go
index 3a0a10333..01538f5ab 100644
--- a/internal/federation/dereferencing/status.go
+++ b/internal/federation/dereferencing/status.go
@@ -951,6 +951,8 @@ func (d *Dereferencer) fetchStatusAttachments(
RemoteURL: &placeholder.RemoteURL,
Description: &placeholder.Description,
Blurhash: &placeholder.Blurhash,
+ FocusX: &placeholder.FileMeta.Focus.X,
+ FocusY: &placeholder.FileMeta.Focus.Y,
},
)
if err != nil {
diff --git a/internal/filter/spam/statusable.go b/internal/filter/spam/statusable.go
index 802accc31..e50753738 100644
--- a/internal/filter/spam/statusable.go
+++ b/internal/filter/spam/statusable.go
@@ -142,7 +142,14 @@ func (f *Filter) StatusableOK(
}
// HEURISTIC 6: Are there any media attachments?
- attachments, _ := ap.ExtractAttachments(statusable)
+ attachments, err := ap.ExtractAttachments(statusable)
+ if err != nil {
+ log.Warnf(ctx,
+ "error(s) extracting attachments for %s: %v",
+ ap.GetJSONLDId(statusable), err,
+ )
+ }
+
hasAttachments := len(attachments) != 0
if hasAttachments {
err := errors.New("status has attachment(s)")
diff --git a/internal/router/template.go b/internal/router/template.go
index a2cb1daf5..0bae96548 100644
--- a/internal/router/template.go
+++ b/internal/router/template.go
@@ -136,6 +136,7 @@ funcMap["includeAttr"] = func(name string, data any) (template.HTMLAttr, error)
var funcMap = template.FuncMap{
"add": add,
"acctInstance": acctInstance,
+ "objectPosition": objectPosition,
"demojify": demojify,
"deref": deref,
"emojify": emojify,
@@ -365,3 +366,12 @@ func deref(i any) any {
return vOf.Elem()
}
+
+// objectPosition formats the given focus coordinates to a
+// string suitable for use as a css object-position value.
+func objectPosition(focusX float32, focusY float32) string {
+ const fmts = "%.2f"
+ xPos := ((focusX / 2) + .5) * 100
+ yPos := ((focusY / -2) + .5) * 100
+ return fmt.Sprintf(fmts, xPos) + "%" + " " + fmt.Sprintf(fmts, yPos) + "%"
+}
diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go
index f21bef83f..debd93a0b 100644
--- a/internal/typeutils/internaltoas.go
+++ b/internal/typeutils/internaltoas.go
@@ -678,22 +678,9 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
status.SetActivityStreamsContent(contentProp)
// attachments
- attachmentProp := streams.NewActivityStreamsAttachmentProperty()
- attachments := s.Attachments
- if len(s.AttachmentIDs) != len(attachments) {
- attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
- if err != nil {
- return nil, gtserror.Newf("error getting attachments from database: %w", err)
- }
+ if err := c.attachAttachments(ctx, s, status); err != nil {
+ return nil, gtserror.Newf("error attaching attachments: %w", err)
}
- for _, a := range attachments {
- doc, err := c.AttachmentToAS(ctx, a)
- if err != nil {
- return nil, gtserror.Newf("error converting attachment: %w", err)
- }
- attachmentProp.AppendActivityStreamsDocument(doc)
- }
- status.SetActivityStreamsAttachment(attachmentProp)
// replies
repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false)
@@ -1130,39 +1117,94 @@ func (c *Converter) EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.Too
return emoji, nil
}
-// AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation
-func (c *Converter) AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error) {
- // type -- Document
- doc := streams.NewActivityStreamsDocument()
+// attachAttachments converts the attachments on the given status
+// into Attachmentables, and appends them to the given Statusable.
+func (c *Converter) attachAttachments(
+ ctx context.Context,
+ s *gtsmodel.Status,
+ statusable ap.Statusable,
+) error {
+ // Ensure status attachments populated.
+ if len(s.AttachmentIDs) != len(s.Attachments) {
+ var err error
+ s.Attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
+ if err != nil && !errors.Is(err, db.ErrNoEntries) {
+ return gtserror.Newf("db error getting attachments: %w", err)
+ }
+ }
- // mediaType aka mime content type
- mediaTypeProp := streams.NewActivityStreamsMediaTypeProperty()
- mediaTypeProp.Set(a.File.ContentType)
- doc.SetActivityStreamsMediaType(mediaTypeProp)
+ // Prepare attachment property.
+ attachmentProp := streams.NewActivityStreamsAttachmentProperty()
+ defer statusable.SetActivityStreamsAttachment(attachmentProp)
+
+ for _, a := range s.Attachments {
+
+ // Use appropriate vocab.Type and
+ // append function for this attachment.
+ var (
+ attachmentable ap.Attachmentable
+ append func()
+ )
+ switch a.Type {
+
+ // png, gif, webp, jpeg, etc.
+ case gtsmodel.FileTypeImage:
+ t := streams.NewActivityStreamsImage()
+ attachmentable = t
+ append = func() { attachmentProp.AppendActivityStreamsImage(t) }
+
+ // mp4, m4a, wmv, webm, etc.
+ case gtsmodel.FileTypeVideo, gtsmodel.FileTypeGifv:
+ t := streams.NewActivityStreamsVideo()
+ attachmentable = t
+ append = func() { attachmentProp.AppendActivityStreamsVideo(t) }
+
+ // mp3, flac, ogg, wma, etc.
+ case gtsmodel.FileTypeAudio:
+ t := streams.NewActivityStreamsAudio()
+ attachmentable = t
+ append = func() { attachmentProp.AppendActivityStreamsAudio(t) }
+
+ // Not sure, fall back to Document.
+ default:
+ t := streams.NewActivityStreamsDocument()
+ attachmentable = t
+ append = func() { attachmentProp.AppendActivityStreamsDocument(t) }
+ }
- // url -- for the original image not the thumbnail
- urlProp := streams.NewActivityStreamsUrlProperty()
- imageURL, err := url.Parse(a.URL)
- if err != nil {
- return nil, fmt.Errorf("AttachmentToAS: error parsing uri %s: %s", a.URL, err)
- }
- urlProp.AppendIRI(imageURL)
- doc.SetActivityStreamsUrl(urlProp)
+ // `mediaType` ie., mime content type.
+ ap.SetMediaType(attachmentable, a.File.ContentType)
- // name -- aka image description
- nameProp := streams.NewActivityStreamsNameProperty()
- nameProp.AppendXMLSchemaString(a.Description)
- doc.SetActivityStreamsName(nameProp)
+ // URL of the media file.
+ imageURL, err := url.Parse(a.URL)
+ if err != nil {
+ return gtserror.Newf("error parsing attachment url: %w", err)
+ }
+ ap.AppendURL(attachmentable, imageURL)
- // blurhash
- blurProp := streams.NewTootBlurhashProperty()
- blurProp.Set(a.Blurhash)
- doc.SetTootBlurhash(blurProp)
+ // `summary` ie., media description / alt text
+ ap.AppendSummary(attachmentable, a.Description)
- // focalpoint
- // TODO
+ // `blurhash`
+ ap.SetBlurhash(attachmentable, a.Blurhash)
- return doc, nil
+ // Set `focalPoint` only if necessary.
+ if a.FileMeta.Focus.X != 0 && a.FileMeta.Focus.Y != 0 {
+ if withFocalPoint, ok := attachmentable.(ap.WithFocalPoint); ok {
+ focalPointProp := streams.NewTootFocalPointProperty()
+ focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.X))
+ focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.Y))
+ withFocalPoint.SetTootFocalPoint(focalPointProp)
+ }
+ }
+
+ // Done, append
+ // to Statusable.
+ append()
+ }
+
+ statusable.SetActivityStreamsAttachment(attachmentProp)
+ return nil
}
// FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.
diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go
index eeba5ac72..f8d69491c 100644
--- a/internal/typeutils/internaltoas_test.go
+++ b/internal/typeutils/internaltoas_test.go
@@ -597,6 +597,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",
"blurhash": "toot:blurhash",
+ "focalPoint": {
+ "@container": "@list",
+ "@id": "toot:focalPoint"
+ },
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#"
}
@@ -604,9 +608,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
"attachment": [
{
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
+ "focalPoint": [
+ -0.5,
+ 0.5
+ ],
"mediaType": "image/jpeg",
- "name": "Black and white image of some 50's style text saying: Welcome On Board",
- "type": "Document",
+ "summary": "Black and white image of some 50's style text saying: Welcome On Board",
+ "type": "Image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
}
],
@@ -697,6 +705,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",
"blurhash": "toot:blurhash",
+ "focalPoint": {
+ "@container": "@list",
+ "@id": "toot:focalPoint"
+ },
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#"
}
@@ -704,9 +716,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
"attachment": [
{
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
+ "focalPoint": [
+ -0.5,
+ 0.5
+ ],
"mediaType": "image/jpeg",
- "name": "Black and white image of some 50's style text saying: Welcome On Board",
- "type": "Document",
+ "summary": "Black and white image of some 50's style text saying: Welcome On Board",
+ "type": "Image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
}
],
diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go
index f5f8ae7ec..63e242f79 100644
--- a/internal/typeutils/internaltofrontend_test.go
+++ b/internal/typeutils/internaltofrontend_test.go
@@ -553,8 +553,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
"aspect": 1.9104477
},
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -701,8 +701,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning
"aspect": 1.9104477
},
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -851,8 +851,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
"aspect": 1.9104477
},
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -1032,8 +1032,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() {
"aspect": 1.9104477
},
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -1218,8 +1218,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
"aspect": 1.9104477
},
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -1955,8 +1955,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
"aspect": 1.9104477
},
"focus": {
- "x": 0,
- "y": 0
+ "x": -0.5,
+ "y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index 7d868ce4d..5f0c2f032 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -709,6 +709,12 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Size: 137216,
Aspect: 1.9104477,
},
+ // Focus on top-left
+ // quadrant of image.
+ Focus: gtsmodel.Focus{
+ X: -0.5,
+ Y: 0.5,
+ },
},
AccountID: "01F8MH17FWEB39HZJ76B6VXSKF",
Description: "Black and white image of some 50's style text saying: Welcome On Board",
diff --git a/web/source/frontend/index.js b/web/source/frontend/index.js
index 47879b2e2..a1c2ca74b 100644
--- a/web/source/frontend/index.js
+++ b/web/source/frontend/index.js
@@ -29,6 +29,7 @@
const Photoswipe = require("photoswipe/dist/umd/photoswipe.umd.min.js");
const PhotoswipeLightbox = require("photoswipe/dist/umd/photoswipe-lightbox.umd.min.js");
const PhotoswipeCaptionPlugin = require("photoswipe-dynamic-caption-plugin").default;
+const ObjectPosition = require("./photoswipe-object-position.js").default;
const Plyr = require("plyr");
const Prism = require("./prism.js");
@@ -61,6 +62,10 @@ new PhotoswipeCaptionPlugin(lightbox, {
}
});
+// Enable object-position plugin for lightbox so that css
+// object-position property can be used on preview images.
+new ObjectPosition(lightbox);
+
lightbox.addFilter('itemData', (item) => {
const el = item.element;
if (
diff --git a/web/source/frontend/photoswipe-object-position.js b/web/source/frontend/photoswipe-object-position.js
new file mode 100644
index 000000000..e67f9737d
--- /dev/null
+++ b/web/source/frontend/photoswipe-object-position.js
@@ -0,0 +1,119 @@
+/*
+ GoToSocial
+ Copyright (C) GoToSocial Authors admin@gotosocial.org
+ SPDX-License-Identifier: AGPL-3.0-or-later
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ Code in this file adapted from:
+ https://github.com/vovayatsyuk/photoswipe-object-position (MIT License).
+*/
+
+function getCroppedBoundsOffset(position, imageSize, thumbSize, zoomLevel) {
+ const float = parseFloat(position);
+ return position.indexOf('%') > 0
+ ? (thumbSize - imageSize * zoomLevel) * float / 100
+ : float;
+}
+
+function getCroppedZoomPan(position, min, max) {
+ const float = parseFloat(position);
+ return position.indexOf('%') > 0 ? min + (max - min) * float / 100 : float;
+}
+
+function getThumbnail(el) {
+ return el.querySelector('img');
+}
+
+function getObjectPosition(el) {
+ return getComputedStyle(el).getPropertyValue('object-position').split(' ');
+}
+
+export default class ObjectPosition {
+ constructor(lightbox) {
+ /**
+ * Make pan adjustments if large image doesn't fit the viewport.
+ *
+ * Examples:
+ * 1. When thumb object-position is 50% 0 (top part is initially visible)
+ * make sure you'll see the top part of the large image as well.
+ * 2. When thumb object-position is 50% 100% (bottom part is initially visible)
+ * make sure you'll see the bottom part of the large image as well.
+ */
+ lightbox.on('initialZoomPan', (event) => {
+ const slide = event.slide;
+ if (!slide.data.element) {
+ // No thumbnail
+ // image set.
+ return;
+ }
+
+ const thumbnailImg = getThumbnail(slide.data.element);
+ if (!thumbnailImg) {
+ // No thumbnail
+ // image set.
+ return;
+ }
+
+ const [positionX, positionY] = getObjectPosition(thumbnailImg);
+
+ if (positionX !== '50%' && slide.pan.x < 0) {
+ slide.pan.x = getCroppedZoomPan(positionX, slide.bounds.min.x, slide.bounds.max.x);
+ }
+
+ if (positionY !== '50%' && slide.pan.y < 0) {
+ slide.pan.y = getCroppedZoomPan(positionY, slide.bounds.min.y, slide.bounds.max.y);
+ }
+ });
+
+ /**
+ * Fix opening animation when thumb object-position is not 50% 50%.
+ * https://github.com/dimsemenov/PhotoSwipe/pull/1868
+ */
+ lightbox.addFilter('thumbBounds', (thumbBounds, itemData) => {
+ if (!itemData.element) {
+ // No thumbnail
+ // image set.
+ return;
+ }
+
+ const thumbnailImg = getThumbnail(itemData.element);
+ if (!thumbnailImg) {
+ // No thumbnail
+ // image set.
+ return;
+ }
+
+ const thumbAreaRect = thumbnailImg.getBoundingClientRect();
+ const fillZoomLevel = thumbBounds.w / itemData.width;
+ const [positionX, positionY] = getObjectPosition(thumbnailImg);
+
+ if (positionX !== '50%') {
+ const offsetX = getCroppedBoundsOffset(positionX, itemData.width, thumbAreaRect.width, fillZoomLevel);
+ thumbBounds.x = thumbAreaRect.left + offsetX;
+ thumbBounds.innerRect.x = offsetX;
+ }
+
+ if (positionY !== '50%') {
+ const offsetY = getCroppedBoundsOffset(positionY, itemData.height, thumbAreaRect.height, fillZoomLevel);
+ thumbBounds.y = thumbAreaRect.top + offsetY;
+ thumbBounds.innerRect.y = offsetY;
+ }
+
+ return thumbBounds;
+ });
+ }
+}
diff --git a/web/source/frontend_prerender/index.js b/web/source/frontend_prerender/index.js
index 294c1ddb1..dd9420655 100644
--- a/web/source/frontend_prerender/index.js
+++ b/web/source/frontend_prerender/index.js
@@ -29,6 +29,11 @@ import { decode } from "blurhash";
const reduceMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
+// Adjust object-position of any image that has a focal point set.
+document.querySelectorAll("img[data-object-position]").forEach(img => {
+ img.style["object-position"] = img.dataset.objectPosition;
+});
+
// Generate a blurhash canvas for each image for
// each blurhash container and put it in the summary.
Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurhashContainer => {
@@ -36,6 +41,7 @@ Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurha
const thumbHeight = blurhashContainer.dataset.blurhashHeight;
const thumbWidth = blurhashContainer.dataset.blurhashWidth;
const thumbAspect = blurhashContainer.dataset.blurhashAspect;
+ const objectPosition = blurhashContainer.dataset.blurhashObjectPosition;
/*
It's very expensive to draw big canvases
@@ -73,6 +79,12 @@ Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurha
imageData.data.set(pixels);
ctx.putImageData(imageData, 0, 0);
+ // Set object-position css property on
+ // the canvas if it's set on the container.
+ if (objectPosition) {
+ canvas.style["object-position"] = objectPosition;
+ }
+
// Put the canvas inside the container.
blurhashContainer.appendChild(canvas);
});
diff --git a/web/source/index.js b/web/source/index.js
index d66afe757..6a218cd08 100644
--- a/web/source/index.js
+++ b/web/source/index.js
@@ -60,7 +60,7 @@ skulk({
transform: [
["babelify", {
global: true,
- ignore: [/node_modules\/(?!(photoswipe.*))/]
+ ignore: [/node_modules\/(?!(.*photoswipe.*))/]
}]
],
},
diff --git a/web/template/status_attachment.tmpl b/web/template/status_attachment.tmpl
index d04f11d5d..6f5041229 100644
--- a/web/template/status_attachment.tmpl
+++ b/web/template/status_attachment.tmpl
@@ -29,6 +29,10 @@
height="{{- .Meta.Small.Height -}}"
data-blurhash-hash="{{- .Blurhash -}}"
data-sensitive="{{- .Sensitive -}}"
+ {{- if or (ne .Meta.Focus.X 0.0) (ne .Meta.Focus.Y 0.0) }}
+ data-object-position="{{ objectPosition .Meta.Focus.X .Meta.Focus.Y }}"
+ {{- else }}
+ {{- end }}
/>
{{- else }}
<img
@@ -69,6 +73,10 @@
data-blurhash-height="{{- .Item.Meta.Small.Height -}}"
data-blurhash-hash="{{- .Item.Blurhash -}}"
data-blurhash-aspect="{{- .Item.Meta.Small.Aspect -}}"
+ {{- if or (ne .Item.Meta.Focus.X 0.0) (ne .Item.Meta.Focus.Y 0.0) }}
+ data-blurhash-object-position="{{ objectPosition .Item.Meta.Focus.X .Item.Meta.Focus.Y }}"
+ {{- else }}
+ {{- end }}
></div>
{{- end }}
</summary>
--
2.49.0
