File tracker-miners-CVE-2023-5557.patch of Package tracker-miners
From 6200816e97d4048c17388e49bb209a0ea9ba7adc Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sat, 23 Sep 2023 10:42:37 +0200
Subject: [PATCH 1/9] tracker-extract: Drop SIGINT/SIGTERM handlers
These bring some questions if using seccomp for the full
process (e.g. requiring additional syscalls, or glib spawning
a thread for it), and are not really mandatory since there's
no requirements for a clean exit.
The only thing that is somewhat lost is ease at valgrinding
with noise from things "definitely lost" in the abrupt termination,
but that does already require manually disabling the seccomp
jail, it's not a big stretch to pile up more local hacks, or
ignore the noise.
---
src/tracker-extract/tracker-main.c | 42 ------------------------------
1 file changed, 42 deletions(-)
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index c1a09685c..eb4e5d0e0 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -129,46 +129,6 @@ initialize_priority_and_scheduling (void)
}
}
-static gboolean
-signal_handler (gpointer user_data)
-{
- int signo = GPOINTER_TO_INT (user_data);
-
- static gboolean in_loop = FALSE;
-
- /* Die if we get re-entrant signals handler calls */
- if (in_loop) {
- _exit (EXIT_FAILURE);
- }
-
- switch (signo) {
- case SIGTERM:
- case SIGINT:
- in_loop = TRUE;
- g_main_loop_quit (main_loop);
-
- /* Fall through */
- default:
- if (g_strsignal (signo)) {
- g_debug ("Received signal:%d->'%s'",
- signo,
- g_strsignal (signo));
- }
- break;
- }
-
- return G_SOURCE_CONTINUE;
-}
-
-static void
-initialize_signal_handler (void)
-{
-#ifndef G_OS_WIN32
- g_unix_signal_add (SIGTERM, signal_handler, GINT_TO_POINTER (SIGTERM));
- g_unix_signal_add (SIGINT, signal_handler, GINT_TO_POINTER (SIGINT));
-#endif /* G_OS_WIN32 */
-}
-
static void
log_option_values (TrackerConfig *config)
{
@@ -467,8 +427,6 @@ main (int argc, char *argv[])
tracker_miner_start (TRACKER_MINER (decorator));
- initialize_signal_handler ();
-
g_main_loop_run (main_loop);
my_main_loop = main_loop;
--
GitLab
From 58c8c12a39cc434bc0a3f384dfd6da53486fa90c Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2023 11:50:46 +0200
Subject: [PATCH 2/9] tracker-extract: Move error report handling to
tracker-miner-fs
These error reports contain useful information in case metadata
extraction failed for some files. However, it would be nice to
have the tracker-extract-3 process uninvolved with file creation.
To make it so for error reports, use a D-Bus signal on a new
interface at the tracker-extract-3 process, that is picked up by
tracker-miner-fs-3. The deletion of stale error reports is already
performed by tracker-miner-fs-3, and the `tracker3 status` CLI as
a last resort.
---
src/libtracker-miner/tracker-decorator.c | 31 +++-----
src/miners/fs/tracker-extract-watchdog.c | 53 ++++++++++++++
.../tracker-extract-controller.c | 72 +++++++++++++++++++
.../tracker-extract-decorator.c | 8 +--
src/tracker-extract/tracker-main.c | 14 ----
5 files changed, 139 insertions(+), 39 deletions(-)
diff --git a/src/libtracker-miner/tracker-decorator.c b/src/libtracker-miner/tracker-decorator.c
index f3f298c04..23cc81bf2 100644
--- a/src/libtracker-miner/tracker-decorator.c
+++ b/src/libtracker-miner/tracker-decorator.c
@@ -102,6 +102,7 @@ enum {
ITEMS_AVAILABLE,
FINISHED,
ERROR,
+ RAISE_ERROR,
LAST_SIGNAL
};
@@ -269,24 +270,6 @@ retry_synchronously (TrackerDecorator *decorator,
}
}
-static void
-tag_success (TrackerDecorator *decorator,
- GArray *commit_buffer)
-{
- guint i;
-
- for (i = 0; i < commit_buffer->len; i++) {
- SparqlUpdate *update;
- GFile *file;
-
- update = &g_array_index (commit_buffer, SparqlUpdate, i);
-
- file = g_file_new_for_uri (update->url);
- tracker_error_report_delete (file);
- g_object_unref (file);
- }
-}
-
static void
decorator_commit_cb (GObject *object,
GAsyncResult *result,
@@ -305,8 +288,6 @@ decorator_commit_cb (GObject *object,
if (!tracker_sparql_connection_update_array_finish (conn, result, NULL)) {
g_debug ("SPARQL error detected in batch, retrying one by one");
retry_synchronously (decorator, priv->commit_buffer);
- } else {
- tag_success (decorator, priv->commit_buffer);
}
g_clear_pointer (&priv->commit_buffer, g_array_unref);
@@ -1080,6 +1061,16 @@ tracker_decorator_class_init (TrackerDecoratorClass *klass)
G_TYPE_STRING,
G_TYPE_STRING,
G_TYPE_STRING);
+
+ signals[RAISE_ERROR] =
+ g_signal_new ("raise-error",
+ G_OBJECT_CLASS_TYPE (object_class),
+ G_SIGNAL_RUN_LAST,
+ 0, NULL, NULL, NULL,
+ G_TYPE_NONE, 3,
+ G_TYPE_FILE,
+ G_TYPE_STRING,
+ G_TYPE_STRING);
}
static void
diff --git a/src/miners/fs/tracker-extract-watchdog.c b/src/miners/fs/tracker-extract-watchdog.c
index 41e12a51c..095ecef00 100644
--- a/src/miners/fs/tracker-extract-watchdog.c
+++ b/src/miners/fs/tracker-extract-watchdog.c
@@ -38,6 +38,7 @@ struct _TrackerExtractWatchdog {
gchar *domain;
guint extractor_watchdog_id;
guint progress_signal_id;
+ guint error_signal_id;
gboolean initializing;
};
@@ -82,6 +83,47 @@ on_extract_progress_cb (GDBusConnection *conn,
status, progress, (gint) remaining);
}
+static void
+on_extract_error_cb (GDBusConnection *conn,
+ const gchar *sender_name,
+ const gchar *object_path,
+ const gchar *interface_name,
+ const gchar *signal_name,
+ GVariant *parameters,
+ gpointer user_data)
+{
+ g_autoptr (GVariant) uri = NULL, message = NULL, extra = NULL, child = NULL;
+ GVariantIter iter;
+ GVariant *value;
+ gchar *key;
+
+ child = g_variant_get_child_value (parameters, 0);
+ g_variant_iter_init (&iter, child);
+
+ while (g_variant_iter_next (&iter, "{sv}", &key, &value)) {
+ if (g_strcmp0 (key, "uri") == 0)
+ uri = g_variant_ref_sink (value);
+ else if (g_strcmp0 (key, "message") == 0)
+ message = g_variant_ref_sink (value);
+ else if (g_strcmp0 (key, "extra-info") == 0)
+ extra = g_variant_ref_sink (value);
+
+ g_variant_unref (value);
+ g_free (key);
+ }
+
+ if (g_variant_is_of_type (uri, G_VARIANT_TYPE_STRING) &&
+ g_variant_is_of_type (message, G_VARIANT_TYPE_STRING) &&
+ (!extra || g_variant_is_of_type (extra, G_VARIANT_TYPE_STRING))) {
+ g_autoptr (GFile) file = NULL;
+
+ file = g_file_new_for_uri (g_variant_get_string (uri, NULL));
+ tracker_error_report (file,
+ g_variant_get_string (message, NULL),
+ extra ? g_variant_get_string (extra, NULL) : NULL);
+ }
+}
+
static void
extract_watchdog_name_appeared (GDBusConnection *conn,
const gchar *name,
@@ -105,6 +147,17 @@ extract_watchdog_name_appeared (GDBusConnection *conn,
on_extract_progress_cb,
watchdog,
NULL);
+ watchdog->error_signal_id =
+ g_dbus_connection_signal_subscribe (watchdog->conn,
+ "org.freedesktop.Tracker3.Miner.Extract",
+ "org.freedesktop.Tracker3.Extract",
+ "Error",
+ "/org/freedesktop/Tracker3/Extract",
+ NULL,
+ G_DBUS_SIGNAL_FLAGS_NONE,
+ on_extract_error_cb,
+ watchdog,
+ NULL);
}
static void
diff --git a/src/tracker-extract/tracker-extract-controller.c b/src/tracker-extract/tracker-extract-controller.c
index d7e79406a..f3d9f7bcb 100644
--- a/src/tracker-extract/tracker-extract-controller.c
+++ b/src/tracker-extract/tracker-extract-controller.c
@@ -33,11 +33,23 @@ struct TrackerExtractControllerPrivate {
TrackerConfig *config;
GCancellable *cancellable;
GDBusConnection *connection;
+ guint object_id;
guint watch_id;
guint progress_signal_id;
gint paused;
};
+#define OBJECT_PATH "/org/freedesktop/Tracker3/Extract"
+
+static const gchar *introspection_xml =
+ "<node>"
+ " <interface name='org.freedesktop.Tracker3.Extract'>"
+ " <signal name='Error'>"
+ " <arg type='a{sv}' name='data' direction='out' />"
+ " </signal>"
+ " </interface>"
+ "</node>";
+
G_DEFINE_TYPE_WITH_PRIVATE (TrackerExtractController, tracker_extract_controller, G_TYPE_OBJECT)
static void
@@ -194,10 +206,52 @@ update_wait_for_miner_fs (TrackerExtractController *self)
}
}
+static void
+decorator_raise_error_cb (TrackerDecorator *decorator,
+ GFile *file,
+ gchar *msg,
+ gchar *extra,
+ TrackerExtractController *controller)
+{
+ TrackerExtractControllerPrivate *priv =
+ tracker_extract_controller_get_instance_private (controller);
+ g_autoptr (GError) error = NULL;
+ g_autofree gchar *uri = NULL;
+ GVariantBuilder builder;
+
+ uri = g_file_get_uri (file);
+
+ g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}"));
+ g_variant_builder_add (&builder, "{sv}", "uri",
+ g_variant_new_string (uri));
+ g_variant_builder_add (&builder, "{sv}", "message",
+ g_variant_new_string (msg));
+
+ if (extra) {
+ g_variant_builder_add (&builder, "{sv}", "extra-info",
+ g_variant_new_string (extra));
+ }
+
+ g_dbus_connection_emit_signal (priv->connection,
+ NULL,
+ OBJECT_PATH,
+ "org.freedesktop.Tracker3.Extract",
+ "Error",
+ g_variant_new ("(@a{sv})", g_variant_builder_end (&builder)),
+ &error);
+
+ if (error)
+ g_warning ("Could not emit signal: %s\n", error->message);
+}
+
static void
tracker_extract_controller_constructed (GObject *object)
{
TrackerExtractController *self = (TrackerExtractController *) object;
+ g_autoptr (GDBusNodeInfo) introspection_data = NULL;
+ GDBusInterfaceVTable interface_vtable = {
+ NULL, NULL, NULL
+ };
G_OBJECT_CLASS (tracker_extract_controller_parent_class)->constructed (object);
@@ -209,6 +263,19 @@ tracker_extract_controller_constructed (GObject *object)
G_CALLBACK (update_wait_for_miner_fs),
self, G_CONNECT_SWAPPED);
update_wait_for_miner_fs (self);
+
+ g_signal_connect (self->priv->decorator, "raise-error",
+ G_CALLBACK (decorator_raise_error_cb), object);
+
+ introspection_data = g_dbus_node_info_new_for_xml (introspection_xml, NULL);
+ g_assert (introspection_data);
+ self->priv->object_id =
+ g_dbus_connection_register_object (self->priv->connection,
+ OBJECT_PATH,
+ introspection_data->interfaces[0],
+ &interface_vtable,
+ object,
+ NULL, NULL);
}
static void
@@ -253,6 +320,11 @@ tracker_extract_controller_dispose (GObject *object)
{
TrackerExtractController *self = (TrackerExtractController *) object;
+ if (self->priv->connection && self->priv->object_id) {
+ g_dbus_connection_unregister_object (self->priv->connection, self->priv->object_id);
+ self->priv->object_id = 0;
+ }
+
disconnect_all (self);
g_clear_object (&self->priv->decorator);
g_clear_object (&self->priv->config);
diff --git a/src/tracker-extract/tracker-extract-decorator.c b/src/tracker-extract/tracker-extract-decorator.c
index f82871655..b5505a466 100644
--- a/src/tracker-extract/tracker-extract-decorator.c
+++ b/src/tracker-extract/tracker-extract-decorator.c
@@ -463,7 +463,7 @@ decorator_ignore_file (GFile *file,
NULL, &error);
if (info) {
- tracker_error_report (file, error_message, extra_info);
+ g_signal_emit_by_name (decorator, "raise-error", file, error_message, extra_info);
mimetype = g_file_info_get_attribute_string (info,
G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE);
@@ -477,10 +477,8 @@ decorator_ignore_file (GFile *file,
} else {
g_debug ("Could not get mimetype: %s", error->message);
- if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
- tracker_error_report_delete (file);
- else
- tracker_error_report (file, error->message, NULL);
+ if (error && !g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+ g_signal_emit_by_name (decorator, "raise-error", file, error_message, extra_info);
g_clear_error (&error);
query = g_strdup_printf ("DELETE {"
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index eb4e5d0e0..422c53044 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -242,15 +242,6 @@ on_decorator_finished (TrackerDecorator *decorator,
main_loop);
}
-static GFile *
-get_cache_dir (TrackerDomainOntology *domain_ontology)
-{
- GFile *cache;
-
- cache = tracker_domain_ontology_get_cache (domain_ontology);
- return g_file_get_child (cache, "files");
-}
-
int
main (int argc, char *argv[])
{
@@ -265,7 +256,6 @@ main (int argc, char *argv[])
TrackerSparqlConnection *sparql_connection;
TrackerDomainOntology *domain_ontology;
gchar *dbus_name, *miner_dbus_name;
- GFile *cache_dir;
bindtextdomain (GETTEXT_PACKAGE, LOCALEDIR);
bind_textdomain_codeset (GETTEXT_PACKAGE, "UTF-8");
@@ -328,10 +318,6 @@ main (int argc, char *argv[])
return EXIT_FAILURE;
}
- cache_dir = get_cache_dir (domain_ontology);
- tracker_error_report_init (cache_dir);
- g_object_unref (cache_dir);
-
config = tracker_config_new ();
/* Extractor command line arguments */
--
GitLab
From e3f4abdd8a77da23e44319294d6fcac231958301 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2023 15:53:22 +0200
Subject: [PATCH 3/9] tracker-extract: Drop handling of wait-for-miner-fs
Even though this setting is off by default, that is the stock
behavior of tracker-miner-fs, by activating the tracker-extract-3
D-Bus name after going idle.
Furthermore, enabling this setting will have clunky interaction
with the current behavior since 3.1.0 that tracker-miner-fs-3
forwards the tracker-extract-3 status (commit bd3ce694d7), since
tracker-extract-3 activity will make the tracker-miner-fs-3 status
"non-idle", which will pause the extractor, which will make the miner
idle, which will unpause the extractor, ...
It's arguable that we should keep supporting this as a setting
altogether, so just drop the tracker-extract-3 side code handling
this setting.
---
.../tracker-extract-controller.c | 166 ------------------
1 file changed, 166 deletions(-)
diff --git a/src/tracker-extract/tracker-extract-controller.c b/src/tracker-extract/tracker-extract-controller.c
index f3d9f7bcb..1543a7237 100644
--- a/src/tracker-extract/tracker-extract-controller.c
+++ b/src/tracker-extract/tracker-extract-controller.c
@@ -30,12 +30,9 @@ enum {
struct TrackerExtractControllerPrivate {
TrackerDecorator *decorator;
- TrackerConfig *config;
GCancellable *cancellable;
GDBusConnection *connection;
guint object_id;
- guint watch_id;
- guint progress_signal_id;
gint paused;
};
@@ -52,160 +49,6 @@ static const gchar *introspection_xml =
G_DEFINE_TYPE_WITH_PRIVATE (TrackerExtractController, tracker_extract_controller, G_TYPE_OBJECT)
-static void
-files_miner_idleness_changed (TrackerExtractController *self,
- gboolean idle)
-{
- if (idle && self->priv->paused) {
- tracker_miner_resume (TRACKER_MINER (self->priv->decorator));
- self->priv->paused = FALSE;
- } else if (!idle && !self->priv->paused) {
- self->priv->paused = FALSE;
- tracker_miner_pause (TRACKER_MINER (self->priv->decorator));
- }
-}
-
-static void
-files_miner_status_changed (TrackerExtractController *self,
- const gchar *status)
-{
- files_miner_idleness_changed (self, g_str_equal (status, "Idle"));
-}
-
-static void
-files_miner_get_status_cb (GObject *source,
- GAsyncResult *result,
- gpointer user_data)
-{
- TrackerExtractController *self = user_data;
- GDBusConnection *conn = (GDBusConnection *) source;
- GVariant *reply;
- const gchar *status;
- GError *error = NULL;
-
- reply = g_dbus_connection_call_finish (conn, result, &error);
- if (!reply) {
- g_debug ("Failed to get tracker-miner-fs status: %s",
- error->message);
- g_clear_error (&error);
- } else {
- g_variant_get (reply, "(&s)", &status);
- files_miner_status_changed (self, status);
- g_variant_unref (reply);
- }
-
- g_clear_object (&self->priv->cancellable);
- g_object_unref (self);
-}
-
-static void
-appeared_cb (GDBusConnection *connection,
- const gchar *name,
- const gchar *name_owner,
- gpointer user_data)
-{
- TrackerExtractController *self = user_data;
-
- /* Get initial status */
- self->priv->cancellable = g_cancellable_new ();
- g_dbus_connection_call (connection,
- "org.freedesktop.Tracker3.Miner.Files",
- "/org/freedesktop/Tracker3/Miner/Files",
- "org.freedesktop.Tracker3.Miner",
- "GetStatus",
- NULL,
- G_VARIANT_TYPE ("(s)"),
- G_DBUS_CALL_FLAGS_NO_AUTO_START,
- -1,
- self->priv->cancellable,
- files_miner_get_status_cb,
- g_object_ref (self));
-}
-
-static void
-vanished_cb (GDBusConnection *connection,
- const gchar *name,
- gpointer user_data)
-{
- TrackerExtractController *self = user_data;
-
- /* tracker-miner-fs vanished, we don't have anything to wait for
- * anymore. */
- files_miner_idleness_changed (self, TRUE);
-}
-
-static void
-files_miner_progress_cb (GDBusConnection *connection,
- const gchar *sender_name,
- const gchar *object_path,
- const gchar *interface_name,
- const gchar *signal_name,
- GVariant *parameters,
- gpointer user_data)
-{
- TrackerExtractController *self = user_data;
- const gchar *status;
-
- g_return_if_fail (g_variant_is_of_type (parameters, G_VARIANT_TYPE ("(sdi)")));
-
- /* If we didn't get the initial status yet, ignore Progress signals */
- if (self->priv->cancellable)
- return;
-
- g_variant_get (parameters, "(&sdi)", &status, NULL, NULL);
- files_miner_status_changed (self, status);
-}
-
-static void
-disconnect_all (TrackerExtractController *self)
-{
- GDBusConnection *conn = self->priv->connection;
-
- if (self->priv->watch_id != 0)
- g_bus_unwatch_name (self->priv->watch_id);
- self->priv->watch_id = 0;
-
- if (self->priv->progress_signal_id != 0)
- g_dbus_connection_signal_unsubscribe (conn,
- self->priv->progress_signal_id);
- self->priv->progress_signal_id = 0;
-
- if (self->priv->cancellable)
- g_cancellable_cancel (self->priv->cancellable);
- g_clear_object (&self->priv->cancellable);
-}
-
-static void
-update_wait_for_miner_fs (TrackerExtractController *self)
-{
- GDBusConnection *conn = self->priv->connection;
-
- if (tracker_config_get_wait_for_miner_fs (self->priv->config)) {
- self->priv->progress_signal_id =
- g_dbus_connection_signal_subscribe (conn,
- "org.freedesktop.Tracker3.Miner.Files",
- "org.freedesktop.Tracker3.Miner",
- "Progress",
- "/org/freedesktop/Tracker3/Miner/Files",
- NULL,
- G_DBUS_SIGNAL_FLAGS_NONE,
- files_miner_progress_cb,
- self, NULL);
-
- /* appeared_cb is guaranteed to be called even if the service
- * was already running, so we'll start the miner from there. */
- self->priv->watch_id = g_bus_watch_name_on_connection (conn,
- "org.freedesktop.Tracker3.Miner.Files",
- G_BUS_NAME_WATCHER_FLAGS_NONE,
- appeared_cb,
- vanished_cb,
- self, NULL);
- } else {
- disconnect_all (self);
- files_miner_idleness_changed (self, TRUE);
- }
-}
-
static void
decorator_raise_error_cb (TrackerDecorator *decorator,
GFile *file,
@@ -257,13 +100,6 @@ tracker_extract_controller_constructed (GObject *object)
g_assert (self->priv->decorator != NULL);
- self->priv->config = g_object_ref (tracker_main_get_config ());
- g_signal_connect_object (self->priv->config,
- "notify::wait-for-miner-fs",
- G_CALLBACK (update_wait_for_miner_fs),
- self, G_CONNECT_SWAPPED);
- update_wait_for_miner_fs (self);
-
g_signal_connect (self->priv->decorator, "raise-error",
G_CALLBACK (decorator_raise_error_cb), object);
@@ -325,9 +161,7 @@ tracker_extract_controller_dispose (GObject *object)
self->priv->object_id = 0;
}
- disconnect_all (self);
g_clear_object (&self->priv->decorator);
- g_clear_object (&self->priv->config);
G_OBJECT_CLASS (tracker_extract_controller_parent_class)->dispose (object);
}
--
GitLab
From 9244bb66bb2f58841dfcdc407136243d1e982c8e Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2023 16:54:39 +0200
Subject: [PATCH 4/9] tracker-extract: Move text allow list handling to
tracker-miner-fs-3
This is something that can be streamlined without delayed handling from
tracker-extract-3. But also avoids having to poke for this setting in
that process.
---
.../tracker-module-manager.c | 35 ++++++++
.../tracker-module-manager.h | 3 +
src/miners/fs/tracker-miner-files.c | 82 +++++++++++++++++++
src/miners/fs/tracker-miner-files.h | 3 +
src/tracker-extract/tracker-extract-text.c | 52 ++++--------
5 files changed, 138 insertions(+), 37 deletions(-)
diff --git a/src/libtracker-extract/tracker-module-manager.c b/src/libtracker-extract/tracker-module-manager.c
index 83c9453e0..e23fab0bc 100644
--- a/src/libtracker-extract/tracker-module-manager.c
+++ b/src/libtracker-extract/tracker-module-manager.c
@@ -408,6 +408,41 @@ tracker_extract_module_manager_get_all_rdf_types (void)
return types;
}
+gboolean
+tracker_extract_module_manager_check_fallback_rdf_type (const gchar *mimetype,
+ const gchar *rdf_type)
+{
+ GList *l, *list;
+ gint i;
+
+ g_return_val_if_fail (mimetype, FALSE);
+ g_return_val_if_fail (rdf_type, FALSE);
+
+ if (!initialized &&
+ !tracker_extract_module_manager_init ()) {
+ return FALSE;
+ }
+
+ list = lookup_rules (mimetype);
+
+ for (l = list; l; l = l->next) {
+ RuleInfo *r_info = l->data;
+
+ if (r_info->fallback_rdf_types == NULL)
+ continue;
+
+ for (i = 0; r_info->fallback_rdf_types[i]; i++) {
+ if (g_strcmp0 (r_info->fallback_rdf_types[i], rdf_type) == 0)
+ return TRUE;
+ }
+
+ /* We only want the first RDF types matching */
+ break;
+ }
+
+ return FALSE;
+}
+
static ModuleInfo *
load_module (RuleInfo *info)
{
diff --git a/src/libtracker-extract/tracker-module-manager.h b/src/libtracker-extract/tracker-module-manager.h
index 0139c11b5..c502dcfd6 100644
--- a/src/libtracker-extract/tracker-module-manager.h
+++ b/src/libtracker-extract/tracker-module-manager.h
@@ -48,6 +48,9 @@ GStrv tracker_extract_module_manager_get_rdf_types (const gchar *mimetype);
const gchar * tracker_extract_module_manager_get_graph (const gchar *mimetype);
const gchar * tracker_extract_module_manager_get_hash (const gchar *mimetype);
+gboolean tracker_extract_module_manager_check_fallback_rdf_type (const gchar *mimetype,
+ const gchar *rdf_type);
+
GModule * tracker_extract_module_manager_get_module (const gchar *mimetype,
const gchar **rule_out,
TrackerExtractMetadataFunc *extract_func_out);
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index a7fed924c..283f1ac36 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -83,6 +83,9 @@ struct TrackerMinerFilesPrivate {
gchar *domain;
TrackerDomainOntology *domain_ontology;
+ GSettings *extract_settings;
+ GStrv allowed_text_patterns;
+
guint disk_space_check_id;
gboolean disk_space_pause;
@@ -123,6 +126,8 @@ enum {
PROP_DOMAIN,
};
+#define TEXT_ALLOWLIST "text-allowlist"
+
static void miner_files_set_property (GObject *object,
guint param_id,
const GValue *value,
@@ -381,6 +386,15 @@ miner_files_initable_iface_init (GInitableIface *iface)
iface->init = miner_files_initable_init;
}
+static void
+text_allowlist_changed_cb (GSettings *settings,
+ const gchar *key,
+ TrackerMinerFiles *mf)
+{
+ g_clear_pointer (&mf->private->allowed_text_patterns, g_strfreev);
+ mf->private->allowed_text_patterns = g_settings_get_strv (settings, TEXT_ALLOWLIST);
+}
+
static gboolean
miner_files_initable_init (GInitable *initable,
GCancellable *cancellable,
@@ -670,6 +684,12 @@ miner_files_initable_init (GInitable *initable,
G_CALLBACK (on_extractor_status), mf);
g_free (domain_name);
+ mf->private->extract_settings = g_settings_new ("org.freedesktop.Tracker3.Extract");
+ g_signal_connect (mf->private->extract_settings, "changed::" TEXT_ALLOWLIST,
+ G_CALLBACK (text_allowlist_changed_cb), mf);
+ mf->private->allowed_text_patterns = g_settings_get_strv (mf->private->extract_settings,
+ TEXT_ALLOWLIST);
+
return TRUE;
}
@@ -735,6 +755,9 @@ miner_files_finalize (GObject *object)
priv->grace_period_timeout_id = 0;
}
+ g_clear_object (&mf->private->extract_settings);
+ g_clear_pointer (&mf->private->allowed_text_patterns, g_strfreev);
+
g_signal_handlers_disconnect_by_func (priv->extract_watchdog,
on_extractor_lost,
NULL);
@@ -2054,6 +2077,31 @@ miner_files_create_folder_information_element (TrackerMinerFiles *miner,
return resource;
}
+static TrackerResource *
+miner_files_create_text_file_information_element (TrackerMinerFiles *miner,
+ GFile *file,
+ const gchar *mime_type,
+ gboolean create)
+{
+ TrackerResource *resource;
+ GStrv rdf_types;
+ const gchar *urn;
+ int i;
+
+ urn = tracker_miner_fs_get_identifier (TRACKER_MINER_FS (miner),
+ file, create, TRUE, NULL);
+ resource = tracker_resource_new (urn);
+
+ rdf_types = tracker_extract_module_manager_get_rdf_types (mime_type);
+
+ for (i = 0; rdf_types[i]; i++)
+ tracker_resource_add_uri (resource, "rdf:type", rdf_types[i]);
+
+ g_strfreev (rdf_types);
+
+ return resource;
+}
+
static void
miner_files_process_file (TrackerMinerFS *fs,
GFile *file,
@@ -2185,6 +2233,21 @@ miner_files_process_file (TrackerMinerFS *fs,
tracker_resource_set_int64 (graph_file, "nfo:fileSize",
g_file_info_get_size (file_info));
+
+ if (tracker_extract_module_manager_check_fallback_rdf_type (mime_type,
+ "nfo:PlainTextDocument") &&
+ !tracker_miner_files_check_allowed_text_file (TRACKER_MINER_FILES (fs), file)) {
+ TrackerResource *text_file;
+
+ /* We let disallowed text files have a shallow nie:InformationElement */
+ text_file = miner_files_create_text_file_information_element (TRACKER_MINER_FILES (fs),
+ file, mime_type, create);
+ tracker_resource_set_take_relation (graph_file, "nie:interpretedAs", text_file);
+ tracker_resource_set_uri (text_file, "nie:isStoredAs", uri);
+
+ tracker_resource_set_string (graph_file, "tracker:extractorHash",
+ tracker_extract_module_manager_get_hash (mime_type));
+ }
}
if (delete_properties_sparql)
@@ -2990,3 +3053,22 @@ tracker_miner_files_set_mtime_checking (TrackerMinerFiles *mf,
{
mf->private->mtime_check = mtime_check;
}
+
+gboolean
+tracker_miner_files_check_allowed_text_file (TrackerMinerFiles *mf,
+ GFile *file)
+{
+ g_autofree gchar *basename = NULL;
+ GStrv text_patterns;
+ int i;
+
+ basename = g_file_get_basename (file);
+ text_patterns = mf->private->allowed_text_patterns;
+
+ for (i = 0; text_patterns && text_patterns[i]; i++) {
+ if (g_pattern_match_simple (text_patterns[i], basename))
+ return TRUE;
+ }
+
+ return FALSE;
+}
diff --git a/src/miners/fs/tracker-miner-files.h b/src/miners/fs/tracker-miner-files.h
index 7198147b1..4c610ed05 100644
--- a/src/miners/fs/tracker-miner-files.h
+++ b/src/miners/fs/tracker-miner-files.h
@@ -77,6 +77,9 @@ void tracker_miner_files_writeback_notify (TrackerMinerFiles *mf,
GFile *file,
const GError *error);
+gboolean tracker_miner_files_check_allowed_text_file (TrackerMinerFiles *mf,
+ GFile *file);
+
G_END_DECLS
#endif /* __TRACKER_MINER_FS_FILES_H__ */
diff --git a/src/tracker-extract/tracker-extract-text.c b/src/tracker-extract/tracker-extract-text.c
index 4d51ec560..ecbfdf4d1 100644
--- a/src/tracker-extract/tracker-extract-text.c
+++ b/src/tracker-extract/tracker-extract-text.c
@@ -40,24 +40,6 @@
#include "tracker-extract.h"
#include "tracker-read.h"
-static gboolean
-allow_file (GSList *text_allowlist_patterns,
- GFile *file)
-{
- GSList *l;
- g_autofree gchar *basename = NULL;
-
- basename = g_file_get_basename (file);
-
- for (l = text_allowlist_patterns; l; l = l->next) {
- if (g_pattern_match_string (l->data, basename)) {
- return TRUE;
- }
- }
-
- return FALSE;
-}
-
static gchar *
get_file_content (GFile *file,
gsize n_bytes,
@@ -101,34 +83,30 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
TrackerResource *metadata;
TrackerConfig *config;
GFile *file;
- GSList *text_allowlist_patterns;
gchar *content = NULL;
GError *inner_error = NULL;
config = tracker_main_get_config ();
- text_allowlist_patterns = tracker_config_get_text_allowlist_patterns (config);
file = tracker_extract_info_get_file (info);
metadata = tracker_resource_new (NULL);
tracker_resource_add_uri (metadata, "rdf:type", "nfo:PlainTextDocument");
- if (allow_file (text_allowlist_patterns, file)) {
- content = get_file_content (tracker_extract_info_get_file (info),
- tracker_config_get_max_bytes (config),
- &inner_error);
-
- if (inner_error != NULL) {
- /* An error occurred, perhaps the file was deleted. */
- g_propagate_prefixed_error (error, inner_error, "Could not open:");
- return FALSE;
- }
-
- if (content) {
- tracker_resource_set_string (metadata, "nie:plainTextContent", content);
- g_free (content);
- } else {
- tracker_resource_set_string (metadata, "nie:plainTextContent", "");
- }
+ content = get_file_content (tracker_extract_info_get_file (info),
+ tracker_config_get_max_bytes (config),
+ &inner_error);
+
+ if (inner_error != NULL) {
+ /* An error occurred, perhaps the file was deleted. */
+ g_propagate_prefixed_error (error, inner_error, "Could not open:");
+ return FALSE;
+ }
+
+ if (content) {
+ tracker_resource_set_string (metadata, "nie:plainTextContent", content);
+ g_free (content);
+ } else {
+ tracker_resource_set_string (metadata, "nie:plainTextContent", "");
}
tracker_extract_info_set_resource (info, metadata);
--
GitLab
From fe729f2d9a693c65e7896eb0c350ea525e7a353d Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2023 14:20:01 +0200
Subject: [PATCH 5/9] tracker-extract: Handle configuration through D-Bus
Add an interface on tracker-miner-fs-3 so that tracker-extract-3
can get the relevant settings without using DConf/GSettings directly.
This replaces all settings usage from tracker-extract-3.
---
src/libtracker-extract/tracker-extract-info.c | 12 +-
src/libtracker-extract/tracker-extract-info.h | 5 +-
src/miners/fs/meson.build | 1 +
src/miners/fs/tracker-files-interface.c | 184 +++++++++++
src/miners/fs/tracker-files-interface.h | 35 +++
src/miners/fs/tracker-main.c | 6 +
src/tracker-extract/meson.build | 1 -
src/tracker-extract/tracker-config.c | 289 ------------------
src/tracker-extract/tracker-config.h | 69 -----
.../tracker-extract-controller.c | 61 ++++
src/tracker-extract/tracker-extract-epub.c | 21 +-
src/tracker-extract/tracker-extract-html.c | 4 +-
.../tracker-extract-msoffice-xml.c | 6 +-
.../tracker-extract-msoffice.c | 4 +-
src/tracker-extract/tracker-extract-oasis.c | 6 +-
src/tracker-extract/tracker-extract-pdf.c | 4 +-
src/tracker-extract/tracker-extract-text.c | 4 +-
src/tracker-extract/tracker-extract.c | 19 +-
src/tracker-extract/tracker-extract.h | 3 +
src/tracker-extract/tracker-main.c | 35 +--
src/tracker-extract/tracker-main.h | 5 -
.../tracker-extract-info-test.c | 4 +-
22 files changed, 343 insertions(+), 435 deletions(-)
create mode 100644 src/miners/fs/tracker-files-interface.c
create mode 100644 src/miners/fs/tracker-files-interface.h
delete mode 100644 src/tracker-extract/tracker-config.c
delete mode 100644 src/tracker-extract/tracker-config.h
diff --git a/src/libtracker-extract/tracker-extract-info.c b/src/libtracker-extract/tracker-extract-info.c
index 46e5fb700..30200a9a6 100644
--- a/src/libtracker-extract/tracker-extract-info.c
+++ b/src/libtracker-extract/tracker-extract-info.c
@@ -45,6 +45,8 @@ struct _TrackerExtractInfo
gchar *mimetype;
gchar *graph;
+ gint max_text;
+
gint ref_count;
};
@@ -66,7 +68,8 @@ G_DEFINE_BOXED_TYPE (TrackerExtractInfo, tracker_extract_info,
TrackerExtractInfo *
tracker_extract_info_new (GFile *file,
const gchar *mimetype,
- const gchar *graph)
+ const gchar *graph,
+ gint max_text)
{
TrackerExtractInfo *info;
@@ -76,6 +79,7 @@ tracker_extract_info_new (GFile *file,
info->file = g_object_ref (file);
info->mimetype = g_strdup (mimetype);
info->graph = g_strdup (graph);
+ info->max_text = max_text;
info->resource = NULL;
@@ -231,3 +235,9 @@ tracker_extract_info_set_resource (TrackerExtractInfo *info,
g_object_ref (resource);
info->resource = resource;
}
+
+gint
+tracker_extract_info_get_max_text (TrackerExtractInfo *info)
+{
+ return info->max_text;
+}
diff --git a/src/libtracker-extract/tracker-extract-info.h b/src/libtracker-extract/tracker-extract-info.h
index 3751c6c94..3de67d181 100644
--- a/src/libtracker-extract/tracker-extract-info.h
+++ b/src/libtracker-extract/tracker-extract-info.h
@@ -37,13 +37,16 @@ GType tracker_extract_info_get_type (void) G_GNUC_
TrackerExtractInfo * tracker_extract_info_new (GFile *file,
const gchar *mimetype,
- const gchar *graph);
+ const gchar *graph,
+ gint max_text);
TrackerExtractInfo * tracker_extract_info_ref (TrackerExtractInfo *info);
void tracker_extract_info_unref (TrackerExtractInfo *info);
GFile * tracker_extract_info_get_file (TrackerExtractInfo *info);
const gchar * tracker_extract_info_get_mimetype (TrackerExtractInfo *info);
const gchar * tracker_extract_info_get_graph (TrackerExtractInfo *info);
+gint tracker_extract_info_get_max_text (TrackerExtractInfo *info);
+
TrackerResource * tracker_extract_info_get_resource (TrackerExtractInfo *info);
void tracker_extract_info_set_resource (TrackerExtractInfo *info,
TrackerResource *resource);
diff --git a/src/miners/fs/meson.build b/src/miners/fs/meson.build
index 84326bef8..57c8ffdc5 100644
--- a/src/miners/fs/meson.build
+++ b/src/miners/fs/meson.build
@@ -1,6 +1,7 @@
sources = [
'tracker-config.c',
'tracker-extract-watchdog.c',
+ 'tracker-files-interface.c',
'tracker-main.c',
'tracker-miner-files.c',
'tracker-storage.c',
diff --git a/src/miners/fs/tracker-files-interface.c b/src/miners/fs/tracker-files-interface.c
new file mode 100644
index 000000000..150e40eab
--- /dev/null
+++ b/src/miners/fs/tracker-files-interface.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2023 Red Hat Inc.
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * Author: Carlos Garnacho <carlosg@gnome.org>
+ */
+
+#include "config-miners.h"
+
+#include "tracker-files-interface.h"
+
+struct _TrackerFilesInterface
+{
+ GObject parent_instance;
+ GDBusConnection *connection;
+ GSettings *settings;
+ guint object_id;
+};
+
+enum {
+ PROP_0,
+ PROP_CONNECTION,
+ N_PROPS,
+};
+
+static GParamSpec *props[N_PROPS] = { 0, };
+
+static const gchar *introspection_xml =
+ "<node>"
+ " <interface name='org.freedesktop.Tracker3.Files'>"
+ " <property name='ExtractorConfig' type='a{sv}' access='read' />"
+ " </interface>"
+ "</node>";
+
+G_DEFINE_TYPE (TrackerFilesInterface, tracker_files_interface, G_TYPE_OBJECT)
+
+static void
+tracker_files_interface_init (TrackerFilesInterface *files_interface)
+{
+}
+
+static GVariant *
+handle_get_property (GDBusConnection *connection,
+ const gchar *sender,
+ const gchar *object_path,
+ const gchar *interface_name,
+ const gchar *property_name,
+ GError **error,
+ gpointer user_data)
+{
+ TrackerFilesInterface *files_interface = user_data;
+
+ if (g_strcmp0 (object_path, "/org/freedesktop/Tracker3/Files") != 0 ||
+ g_strcmp0 (interface_name, "org.freedesktop.Tracker3.Files") != 0) {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA,
+ "Wrong object/interface");
+ return NULL;
+ }
+
+ if (g_strcmp0 (property_name, "ExtractorConfig") == 0) {
+ GVariantBuilder builder;
+
+ g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}"));
+ g_variant_builder_add (&builder, "{sv}", "max-bytes",
+ g_settings_get_value (files_interface->settings, "max-bytes"));
+
+ return g_variant_builder_end (&builder);
+ } else {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA,
+ "Unknown property");
+ return NULL;
+ }
+}
+
+static void
+tracker_files_interface_constructed (GObject *object)
+{
+ TrackerFilesInterface *files_interface = TRACKER_FILES_INTERFACE (object);
+ GDBusInterfaceVTable vtable = { NULL, handle_get_property, NULL };
+ g_autoptr (GDBusNodeInfo) introspection_data = NULL;
+
+ G_OBJECT_CLASS (tracker_files_interface_parent_class)->constructed (object);
+
+ introspection_data = g_dbus_node_info_new_for_xml (introspection_xml, NULL);
+ files_interface->object_id =
+ g_dbus_connection_register_object (files_interface->connection,
+ "/org/freedesktop/Tracker3/Files",
+ introspection_data->interfaces[0],
+ &vtable, object, NULL, NULL);
+
+ files_interface->settings = g_settings_new ("org.freedesktop.Tracker3.Extract");
+}
+
+static void
+tracker_files_interface_finalize (GObject *object)
+{
+ TrackerFilesInterface *files_interface = TRACKER_FILES_INTERFACE (object);
+
+ g_dbus_connection_unregister_object (files_interface->connection,
+ files_interface->object_id);
+ g_clear_object (&files_interface->connection);
+ g_clear_object (&files_interface->settings);
+
+ G_OBJECT_CLASS (tracker_files_interface_parent_class)->finalize (object);
+}
+
+static void
+tracker_files_interface_set_property (GObject *object,
+ guint prop_id,
+ const GValue *value,
+ GParamSpec *pspec)
+{
+ TrackerFilesInterface *files_interface = TRACKER_FILES_INTERFACE (object);
+
+ switch (prop_id) {
+ case PROP_CONNECTION:
+ files_interface->connection = g_value_dup_object (value);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+tracker_files_interface_get_property (GObject *object,
+ guint prop_id,
+ GValue *value,
+ GParamSpec *pspec)
+{
+ TrackerFilesInterface *files_interface = TRACKER_FILES_INTERFACE (object);
+
+ switch (prop_id) {
+ case PROP_CONNECTION:
+ g_value_set_object (value, files_interface->connection);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+tracker_files_interface_class_init (TrackerFilesInterfaceClass *klass)
+{
+ GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+ object_class->constructed = tracker_files_interface_constructed;
+ object_class->finalize = tracker_files_interface_finalize;
+ object_class->set_property = tracker_files_interface_set_property;
+ object_class->get_property = tracker_files_interface_get_property;
+
+ props[PROP_CONNECTION] =
+ g_param_spec_object ("connection",
+ NULL, NULL,
+ G_TYPE_DBUS_CONNECTION,
+ G_PARAM_READWRITE |
+ G_PARAM_CONSTRUCT_ONLY |
+ G_PARAM_STATIC_STRINGS);
+
+ g_object_class_install_properties (object_class, N_PROPS, props);
+}
+
+TrackerFilesInterface *
+tracker_files_interface_new (GDBusConnection *connection)
+{
+ return g_object_new (TRACKER_TYPE_FILES_INTERFACE,
+ "connection", connection,
+ NULL);
+}
diff --git a/src/miners/fs/tracker-files-interface.h b/src/miners/fs/tracker-files-interface.h
new file mode 100644
index 000000000..e040e41d0
--- /dev/null
+++ b/src/miners/fs/tracker-files-interface.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2023 Red Hat Inc.
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * Author: Carlos Garnacho <carlosg@gnome.org>
+ */
+
+#ifndef __TRACKER_FILES_INTERFACE_H__
+#define __TRACKER_FILES_INTERFACE_H__
+
+#include <gio/gio.h>
+
+#define TRACKER_TYPE_FILES_INTERFACE (tracker_files_interface_get_type ())
+G_DECLARE_FINAL_TYPE (TrackerFilesInterface,
+ tracker_files_interface,
+ TRACKER, FILES_INTERFACE,
+ GObject)
+
+TrackerFilesInterface * tracker_files_interface_new (GDBusConnection *connection);
+
+#endif /* __TRACKER_FILES_INTERFACE_H__ */
diff --git a/src/miners/fs/tracker-main.c b/src/miners/fs/tracker-main.c
index 52cb4ed7c..e48fe6f27 100644
--- a/src/miners/fs/tracker-main.c
+++ b/src/miners/fs/tracker-main.c
@@ -41,6 +41,7 @@
#include "tracker-config.h"
#include "tracker-miner-files.h"
+#include "tracker-files-interface.h"
#define ABOUT \
"Tracker " PACKAGE_VERSION "\n"
@@ -967,6 +968,7 @@ main (gint argc, gchar *argv[])
GMemoryMonitor *memory_monitor;
#endif
gchar *domain_name, *dbus_name;
+ TrackerFilesInterface *files_interface;
main_loop = NULL;
@@ -1022,6 +1024,8 @@ main (gint argc, gchar *argv[])
return EXIT_FAILURE;
}
+ files_interface = tracker_files_interface_new (connection);
+
/* Initialize logging */
config = tracker_config_new ();
@@ -1188,6 +1192,8 @@ main (gint argc, gchar *argv[])
save_current_locale (domain_ontology);
}
+ g_object_unref (files_interface);
+
g_main_loop_unref (main_loop);
g_object_unref (config);
diff --git a/src/tracker-extract/meson.build b/src/tracker-extract/meson.build
index 88ce49fb9..0e1f92fe7 100644
--- a/src/tracker-extract/meson.build
+++ b/src/tracker-extract/meson.build
@@ -136,7 +136,6 @@ tracker_extract_priority_dbus = gnome.gdbus_codegen(
namespace: 'TrackerExtractDBus')
tracker_extract_sources = [
- 'tracker-config.c',
'tracker-extract.c',
'tracker-extract-controller.c',
'tracker-extract-decorator.c',
diff --git a/src/tracker-extract/tracker-config.c b/src/tracker-extract/tracker-config.c
deleted file mode 100644
index 426bba67c..000000000
--- a/src/tracker-extract/tracker-config.c
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Copyright (C) 2009, Nokia <ivan.frade@nokia.com>
- * Copyright (C) 2014, Lanedo <martyn@lanedo.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#include "config-miners.h"
-
-#define G_SETTINGS_ENABLE_BACKEND
-#include <gio/gsettingsbackend.h>
-
-#include <libtracker-miners-common/tracker-common.h>
-
-#include "tracker-config.h"
-
-#define CONFIG_SCHEMA "org.freedesktop.Tracker3.Extract"
-#define CONFIG_PATH "/org/freedesktop/tracker/extract/"
-
-static void config_set_property (GObject *object,
- guint param_id,
- const GValue *value,
- GParamSpec *pspec);
-static void config_get_property (GObject *object,
- guint param_id,
- GValue *value,
- GParamSpec *pspec);
-static void config_finalize (GObject *object);
-static void config_constructed (GObject *object);
-
-enum {
- PROP_0,
- PROP_MAX_BYTES,
- PROP_TEXT_ALLOWLIST,
- PROP_WAIT_FOR_MINER_FS,
-};
-
-G_DEFINE_TYPE (TrackerConfig, tracker_config, G_TYPE_SETTINGS);
-
-static void
-tracker_config_class_init (TrackerConfigClass *klass)
-{
- GObjectClass *object_class = G_OBJECT_CLASS (klass);
-
- object_class->set_property = config_set_property;
- object_class->get_property = config_get_property;
- object_class->finalize = config_finalize;
- object_class->constructed = config_constructed;
-
- /* General */
- g_object_class_install_property (object_class,
- PROP_MAX_BYTES,
- g_param_spec_int ("max-bytes",
- "Max Bytes",
- "Maximum number of UTF-8 bytes to extract per file [0->10485760]",
- 0, 1024 * 1024 * 10,
- 1024 * 1024,
- G_PARAM_READWRITE));
-
- g_object_class_install_property (object_class,
- PROP_TEXT_ALLOWLIST,
- g_param_spec_boxed ("text-allowlist",
- "Text file allowlist",
- "Filename patterns for plain text documents that should be indexed",
- G_TYPE_STRV,
- G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
- g_object_class_install_property (object_class,
- PROP_WAIT_FOR_MINER_FS,
- g_param_spec_boolean ("wait-for-miner-fs",
- "Wait for FS miner to be done before extracting",
- "%TRUE to wait for tracker-miner-fs is done before extracting. %FAlSE otherwise",
- FALSE,
- G_PARAM_READWRITE));
-}
-
-static void
-tracker_config_init (TrackerConfig *object)
-{
-}
-
-static void
-config_set_property (GObject *object,
- guint param_id,
- const GValue *value,
- GParamSpec *pspec)
-{
- switch (param_id) {
- /* We don't care about these... we don't save anyway. */
- case PROP_MAX_BYTES:
- case PROP_TEXT_ALLOWLIST:
- case PROP_WAIT_FOR_MINER_FS:
- break;
-
- default:
- G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
- break;
- };
-}
-
-static void
-config_get_property (GObject *object,
- guint param_id,
- GValue *value,
- GParamSpec *pspec)
-{
- TrackerConfig *config = TRACKER_CONFIG (object);
-
- switch (param_id) {
- case PROP_MAX_BYTES:
- g_value_set_int (value,
- tracker_config_get_max_bytes (config));
- break;
-
- case PROP_TEXT_ALLOWLIST:
- g_value_take_boxed (value, tracker_gslist_to_string_list (config->text_allowlist));
- break;
-
- case PROP_WAIT_FOR_MINER_FS:
- g_value_set_boolean (value,
- tracker_config_get_wait_for_miner_fs (config));
- break;
-
- default:
- G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
- break;
- };
-}
-
-static void
-config_set_text_allowlist_conveniences (TrackerConfig *config)
-{
- GSList *l;
- GSList *patterns = NULL;
-
- g_slist_foreach (config->text_allowlist_patterns,
- (GFunc) g_pattern_spec_free,
- NULL);
- g_slist_free (config->text_allowlist_patterns);
-
- for (l = config->text_allowlist; l; l = l->next) {
- GPatternSpec *spec;
- const gchar *str = l->data;
-
- if (str) {
- spec = g_pattern_spec_new (l->data);
- patterns = g_slist_prepend (patterns, spec);
- }
- }
-
- config->text_allowlist_patterns = g_slist_reverse (patterns);
-}
-
-static void
-config_finalize (GObject *object)
-{
- TrackerConfig *config = TRACKER_CONFIG (object);
-
- g_slist_foreach (config->text_allowlist_patterns,
- (GFunc) g_pattern_spec_free,
- NULL);
- g_slist_free (config->text_allowlist);
-
- (G_OBJECT_CLASS (tracker_config_parent_class)->finalize) (object);
-
-}
-
-static void
-config_constructed (GObject *object)
-{
- GSettings *settings;
-
- (G_OBJECT_CLASS (tracker_config_parent_class)->constructed) (object);
-
- settings = G_SETTINGS (object);
-
- if (G_LIKELY (!g_getenv ("TRACKER_USE_CONFIG_FILES"))) {
- g_settings_delay (settings);
- }
-
- /* Set up bindings:
- *
- * We don't bind the G_SETTINGS_BIND_SET because we don't want to save
- * anything, ever, we only want to know about updates to the settings as
- * they're changed externally. The only time this may be
- * different is where we use the environment variable
- * TRACKER_USE_CONFIG_FILES and we want to write a config
- * file for convenience. But this is only necessary if the
- * config is different to the default.
- */
- g_settings_bind (settings, "wait-for-miner-fs", object, "wait-for-miner-fs", G_SETTINGS_BIND_GET);
-
- /* Cache settings accessed from extractor modules, we don't want
- * the GSettings object accessed within these as it may trigger
- * unintended open() calls.
- */
- TRACKER_CONFIG (settings)->max_bytes = g_settings_get_int (settings, "max-bytes");
- TRACKER_CONFIG (settings)->text_allowlist = tracker_string_list_to_gslist (g_settings_get_strv (settings, "text-allowlist"), -1);
-
- config_set_text_allowlist_conveniences (TRACKER_CONFIG (settings));
-}
-
-TrackerConfig *
-tracker_config_new (void)
-{
- TrackerConfig *config = NULL;
-
- /* FIXME: should we unset GSETTINGS_BACKEND env var? */
-
- if (G_UNLIKELY (g_getenv ("TRACKER_USE_CONFIG_FILES"))) {
- GSettingsBackend *backend;
- gchar *filename, *basename;
- gboolean need_to_save;
-
- basename = g_strdup_printf ("%s.cfg", g_get_prgname ());
- filename = g_build_filename (g_get_user_config_dir (), "tracker", basename, NULL);
- g_free (basename);
-
- need_to_save = g_file_test (filename, G_FILE_TEST_EXISTS) == FALSE;
-
- backend = g_keyfile_settings_backend_new (filename, CONFIG_PATH, "General");
- g_info ("Using config file '%s'", filename);
- g_free (filename);
-
- config = g_object_new (TRACKER_TYPE_CONFIG,
- "backend", backend,
- "schema-id", CONFIG_SCHEMA,
- "path", CONFIG_PATH,
- NULL);
- g_object_unref (backend);
-
- if (need_to_save) {
- g_info (" Config file does not exist, using default values...");
- }
- } else {
- config = g_object_new (TRACKER_TYPE_CONFIG,
- "schema-id", CONFIG_SCHEMA,
- "path", CONFIG_PATH,
- NULL);
- }
-
- return config;
-}
-
-gint
-tracker_config_get_max_bytes (TrackerConfig *config)
-{
- g_return_val_if_fail (TRACKER_IS_CONFIG (config), 0);
-
- return config->max_bytes;
-}
-
-GSList *
-tracker_config_get_text_allowlist (TrackerConfig *config)
-{
- g_return_val_if_fail (TRACKER_IS_CONFIG (config), NULL);
-
- return config->text_allowlist;
-}
-
-gboolean
-tracker_config_get_wait_for_miner_fs (TrackerConfig *config)
-{
- g_return_val_if_fail (TRACKER_IS_CONFIG (config), FALSE);
-
- return g_settings_get_boolean (G_SETTINGS (config), "wait-for-miner-fs");
-}
-
-
-/*
- * Convenience functions
- */
-GSList *
-tracker_config_get_text_allowlist_patterns (TrackerConfig *config)
-{
- return config->text_allowlist_patterns;
-}
diff --git a/src/tracker-extract/tracker-config.h b/src/tracker-extract/tracker-config.h
deleted file mode 100644
index 18dc292b0..000000000
--- a/src/tracker-extract/tracker-config.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (C) 2009, Nokia <ivan.frade@nokia.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#ifndef __TRACKER_EXTRACT_CONFIG_H__
-#define __TRACKER_EXTRACT_CONFIG_H__
-
-#include <glib-object.h>
-
-G_BEGIN_DECLS
-
-#define TRACKER_TYPE_CONFIG (tracker_config_get_type ())
-#define TRACKER_CONFIG(o) (G_TYPE_CHECK_INSTANCE_CAST ((o), TRACKER_TYPE_CONFIG, TrackerConfig))
-#define TRACKER_CONFIG_CLASS(k) (G_TYPE_CHECK_CLASS_CAST ((k), TRACKER_TYPE_CONFIG, TrackerConfigClass))
-#define TRACKER_IS_CONFIG(o) (G_TYPE_CHECK_INSTANCE_TYPE ((o), TRACKER_TYPE_CONFIG))
-#define TRACKER_IS_CONFIG_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE ((k), TRACKER_TYPE_CONFIG))
-#define TRACKER_CONFIG_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), TRACKER_TYPE_CONFIG, TrackerConfigClass))
-
-typedef struct TrackerConfig TrackerConfig;
-typedef struct TrackerConfigClass TrackerConfigClass;
-
-struct TrackerConfig {
- GSettings parent;
- gint max_bytes;
- GSList *text_allowlist;
-
- /* Convenience data */
- GSList *text_allowlist_patterns;
-};
-
-struct TrackerConfigClass {
- GSettingsClass parent_class;
-};
-
-GType tracker_config_get_type (void) G_GNUC_CONST;
-
-TrackerConfig *tracker_config_new (void);
-gint tracker_config_get_max_bytes (TrackerConfig *config);
-GSList * tracker_config_get_text_allowlist (TrackerConfig *config);
-gboolean tracker_config_get_wait_for_miner_fs (TrackerConfig *config);
-
-/*
- * Convenience functions:
- */
-
-/* The _patterns() APIs return GPatternSpec pointers for basename
- * pattern matching.
- */
-GSList * tracker_config_get_text_allowlist_patterns (TrackerConfig *config);
-
-G_END_DECLS
-
-#endif /* __TRACKER_EXTRACT_CONFIG_H__ */
-
diff --git a/src/tracker-extract/tracker-extract-controller.c b/src/tracker-extract/tracker-extract-controller.c
index 1543a7237..404f1f13c 100644
--- a/src/tracker-extract/tracker-extract-controller.c
+++ b/src/tracker-extract/tracker-extract-controller.c
@@ -32,6 +32,7 @@ struct TrackerExtractControllerPrivate {
TrackerDecorator *decorator;
GCancellable *cancellable;
GDBusConnection *connection;
+ GDBusProxy *miner_proxy;
guint object_id;
gint paused;
};
@@ -49,6 +50,53 @@ static const gchar *introspection_xml =
G_DEFINE_TYPE_WITH_PRIVATE (TrackerExtractController, tracker_extract_controller, G_TYPE_OBJECT)
+static void
+update_extract_config (TrackerExtractController *controller,
+ GDBusProxy *proxy)
+{
+ TrackerExtractControllerPrivate *priv;
+ GVariantIter iter;
+ g_autoptr (GVariant) v = NULL;
+ GVariant *value;
+ gchar *key;
+
+ priv = tracker_extract_controller_get_instance_private (controller);
+
+ v = g_dbus_proxy_get_cached_property (proxy, "ExtractorConfig");
+ if (!v)
+ return;
+
+ g_variant_iter_init (&iter, v);
+
+ while (g_variant_iter_next (&iter, "{sv}", &key, &value)) {
+ if (g_strcmp0 (key, "max-bytes") == 0 &&
+ g_variant_is_of_type (value, G_VARIANT_TYPE_INT32)) {
+ TrackerExtract *extract = NULL;
+ gint max_bytes;
+
+ max_bytes = g_variant_get_int32 (value);
+ g_object_get (priv->decorator, "extractor", &extract, NULL);
+
+ if (extract) {
+ tracker_extract_set_max_text (extract, max_bytes);
+ g_object_unref (extract);
+ }
+ }
+
+ g_free (key);
+ g_variant_unref (value);
+ }
+}
+
+static void
+miner_properties_changed_cb (GDBusProxy *proxy,
+ GVariant *changed_properties,
+ GStrv invalidated_properties,
+ gpointer user_data)
+{
+ update_extract_config (user_data, proxy);
+}
+
static void
decorator_raise_error_cb (TrackerDecorator *decorator,
GFile *file,
@@ -112,6 +160,19 @@ tracker_extract_controller_constructed (GObject *object)
&interface_vtable,
object,
NULL, NULL);
+
+ self->priv->miner_proxy = g_dbus_proxy_new_sync (self->priv->connection,
+ G_DBUS_PROXY_FLAGS_DO_NOT_AUTO_START,
+ NULL,
+ "org.freedesktop.Tracker3.Miner.Files",
+ "/org/freedesktop/Tracker3/Files",
+ "org.freedesktop.Tracker3.Files",
+ NULL, NULL);
+ if (self->priv->miner_proxy) {
+ g_signal_connect (self->priv->miner_proxy, "g-properties-changed",
+ G_CALLBACK (miner_properties_changed_cb), object);
+ update_extract_config (self, self->priv->miner_proxy);
+ }
}
static void
diff --git a/src/tracker-extract/tracker-extract-epub.c b/src/tracker-extract/tracker-extract-epub.c
index a6bcdcba6..dbf314d94 100644
--- a/src/tracker-extract/tracker-extract-epub.c
+++ b/src/tracker-extract/tracker-extract-epub.c
@@ -561,12 +561,12 @@ extract_opf_path (const gchar *uri)
}
static gchar *
-extract_opf_contents (const gchar *uri,
- const gchar *content_prefix,
- GList *content_files)
+extract_opf_contents (TrackerExtractInfo *info,
+ const gchar *uri,
+ const gchar *content_prefix,
+ GList *content_files)
{
OPFContentData content_data = { 0 };
- TrackerConfig *config;
GError *error = NULL;
GList *l;
GMarkupParser xml_parser = {
@@ -575,10 +575,8 @@ extract_opf_contents (const gchar *uri,
NULL, NULL
};
- config = tracker_main_get_config ();
-
content_data.contents = g_string_new ("");
- content_data.limit = (gsize) tracker_config_get_max_bytes (config);
+ content_data.limit = (gsize) tracker_extract_info_get_max_text (info);
g_debug ("Extracting up to %" G_GSIZE_FORMAT " bytes of content", content_data.limit);
@@ -611,8 +609,9 @@ extract_opf_contents (const gchar *uri,
}
static TrackerResource *
-extract_opf (const gchar *uri,
- const gchar *opf_path)
+extract_opf (TrackerExtractInfo *info,
+ const gchar *uri,
+ const gchar *opf_path)
{
TrackerResource *ebook;
GMarkupParseContext *context;
@@ -652,7 +651,7 @@ extract_opf (const gchar *uri,
}
dirname = g_path_get_dirname (opf_path);
- contents = extract_opf_contents (uri, dirname, data->pages);
+ contents = extract_opf_contents (info, uri, dirname, data->pages);
g_free (dirname);
if (contents && *contents) {
@@ -683,7 +682,7 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
return FALSE;
}
- ebook = extract_opf (uri, opf_path);
+ ebook = extract_opf (info, uri, opf_path);
g_free (opf_path);
g_free (uri);
diff --git a/src/tracker-extract/tracker-extract-html.c b/src/tracker-extract/tracker-extract-html.c
index 76e31befc..dac02f669 100644
--- a/src/tracker-extract/tracker-extract-html.c
+++ b/src/tracker-extract/tracker-extract-html.c
@@ -234,7 +234,6 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
{
TrackerResource *metadata;
GFile *file;
- TrackerConfig *config;
htmlDocPtr doc;
parser_data pd;
gchar *filename;
@@ -284,8 +283,7 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
pd.plain_text = g_string_new (NULL);
pd.title = g_string_new (NULL);
- config = tracker_main_get_config ();
- pd.n_bytes_remaining = tracker_config_get_max_bytes (config);
+ pd.n_bytes_remaining = tracker_extract_info_get_max_text (info);
filename = g_file_get_path (file);
doc = htmlSAXParseFile (filename, NULL, &handler, &pd);
diff --git a/src/tracker-extract/tracker-extract-msoffice-xml.c b/src/tracker-extract/tracker-extract-msoffice-xml.c
index 31d8a7880..86f48822e 100644
--- a/src/tracker-extract/tracker-extract-msoffice-xml.c
+++ b/src/tracker-extract/tracker-extract-msoffice-xml.c
@@ -806,7 +806,6 @@ tracker_extract_get_metadata (TrackerExtractInfo *extract_info,
MsOfficeXMLParserInfo info = { 0 };
MsOfficeXMLFileType file_type;
TrackerResource *metadata;
- TrackerConfig *config;
GMarkupParseContext *context = NULL;
GError *inner_error = NULL;
GFile *file;
@@ -822,9 +821,6 @@ tracker_extract_get_metadata (TrackerExtractInfo *extract_info,
/* Get current Content Type */
file_type = msoffice_xml_get_file_type (uri);
- /* Setup conf */
- config = tracker_main_get_config ();
-
g_debug ("Extracting MsOffice XML format...");
metadata = tracker_resource_new (NULL);
@@ -840,7 +836,7 @@ tracker_extract_get_metadata (TrackerExtractInfo *extract_info,
info.content = NULL;
info.title_already_set = FALSE;
info.generator_already_set = FALSE;
- info.bytes_pending = tracker_config_get_max_bytes (config);
+ info.bytes_pending = tracker_extract_info_get_max_text (extract_info);
/* Create content-type parser context */
context = g_markup_parse_context_new (&content_types_parser,
diff --git a/src/tracker-extract/tracker-extract-msoffice.c b/src/tracker-extract/tracker-extract-msoffice.c
index 06220671b..92f81f9de 100644
--- a/src/tracker-extract/tracker-extract-msoffice.c
+++ b/src/tracker-extract/tracker-extract-msoffice.c
@@ -1619,7 +1619,6 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
GError **error)
{
TrackerResource *metadata;
- TrackerConfig *config;
GsfInfile *infile = NULL;
gchar *content = NULL, *uri;
gboolean is_encrypted = FALSE;
@@ -1669,8 +1668,7 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
extract_summary (metadata, infile, uri);
/* Set max bytes to read from content */
- config = tracker_main_get_config ();
- max_bytes = tracker_config_get_max_bytes (config);
+ max_bytes = tracker_extract_info_get_max_text (info);
if (g_ascii_strcasecmp (mime_used, "application/msword") == 0) {
/* Word file */
diff --git a/src/tracker-extract/tracker-extract-oasis.c b/src/tracker-extract/tracker-extract-oasis.c
index b29ae5971..3d2a4fa46 100644
--- a/src/tracker-extract/tracker-extract-oasis.c
+++ b/src/tracker-extract/tracker-extract-oasis.c
@@ -171,7 +171,6 @@ tracker_extract_get_metadata (TrackerExtractInfo *extract_info,
GError **error)
{
TrackerResource *metadata;
- TrackerConfig *config;
ODTMetadataParseInfo info = { 0 };
ODTFileType file_type;
GFile *file;
@@ -196,9 +195,6 @@ tracker_extract_get_metadata (TrackerExtractInfo *extract_info,
file = tracker_extract_info_get_file (extract_info);
uri = g_file_get_uri (file);
- /* Setup conf */
- config = tracker_main_get_config ();
-
g_debug ("Extracting OASIS metadata and contents from '%s'", uri);
/* First, parse metadata */
@@ -233,7 +229,7 @@ tracker_extract_get_metadata (TrackerExtractInfo *extract_info,
/* Extract content with the given limitations */
extract_oasis_content (uri,
- tracker_config_get_max_bytes (config),
+ tracker_extract_info_get_max_text (extract_info),
file_type,
metadata);
diff --git a/src/tracker-extract/tracker-extract-pdf.c b/src/tracker-extract/tracker-extract-pdf.c
index 40c0c98e4..4706f082a 100644
--- a/src/tracker-extract/tracker-extract-pdf.c
+++ b/src/tracker-extract/tracker-extract-pdf.c
@@ -285,7 +285,6 @@ G_MODULE_EXPORT gboolean
tracker_extract_get_metadata (TrackerExtractInfo *info,
GError **error)
{
- TrackerConfig *config;
time_t creation_date;
GError *inner_error = NULL;
TrackerResource *metadata;
@@ -541,8 +540,7 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
tracker_resource_set_int64 (metadata, "nfo:pageCount", poppler_document_get_n_pages(document));
- config = tracker_main_get_config ();
- n_bytes = tracker_config_get_max_bytes (config);
+ n_bytes = tracker_extract_info_get_max_text (info);
content = extract_content_text (document, n_bytes);
if (content) {
diff --git a/src/tracker-extract/tracker-extract-text.c b/src/tracker-extract/tracker-extract-text.c
index ecbfdf4d1..c2ef33fc6 100644
--- a/src/tracker-extract/tracker-extract-text.c
+++ b/src/tracker-extract/tracker-extract-text.c
@@ -81,19 +81,17 @@ tracker_extract_get_metadata (TrackerExtractInfo *info,
GError **error)
{
TrackerResource *metadata;
- TrackerConfig *config;
GFile *file;
gchar *content = NULL;
GError *inner_error = NULL;
- config = tracker_main_get_config ();
file = tracker_extract_info_get_file (info);
metadata = tracker_resource_new (NULL);
tracker_resource_add_uri (metadata, "rdf:type", "nfo:PlainTextDocument");
content = get_file_content (tracker_extract_info_get_file (info),
- tracker_config_get_max_bytes (config),
+ tracker_extract_info_get_max_text (info),
&inner_error);
if (inner_error != NULL) {
diff --git a/src/tracker-extract/tracker-extract.c b/src/tracker-extract/tracker-extract.c
index 76ed041f1..749e73826 100644
--- a/src/tracker-extract/tracker-extract.c
+++ b/src/tracker-extract/tracker-extract.c
@@ -47,6 +47,8 @@ G_DEFINE_QUARK (TrackerExtractError, tracker_extract_error)
#define DEADLINE_SECONDS 30
+#define DEFAULT_MAX_TEXT 1048576
+
extern gboolean debug;
typedef struct {
@@ -59,6 +61,8 @@ typedef struct {
GHashTable *statistics_data;
GList *running_tasks;
+ gint max_text;
+
/* used to maintain the running tasks
* and stats from different threads
*/
@@ -84,6 +88,7 @@ typedef struct {
gchar *file;
gchar *mimetype;
const gchar *graph;
+ gint max_text;
TrackerExtractMetadataFunc func;
GModule *module;
@@ -124,6 +129,7 @@ tracker_extract_init (TrackerExtract *object)
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
priv->single_thread_extractors = g_hash_table_new (NULL, NULL);
+ priv->max_text = DEFAULT_MAX_TEXT;
#ifdef G_ENABLE_DEBUG
if (TRACKER_DEBUG_CHECK (STATISTICS)) {
@@ -287,7 +293,7 @@ get_file_metadata (TrackerExtractTask *task,
*info_out = NULL;
file = g_file_new_for_uri (task->file);
- info = tracker_extract_info_new (file, task->mimetype, task->graph);
+ info = tracker_extract_info_new (file, task->mimetype, task->graph, task->max_text);
g_object_unref (file);
if (!task->mimetype || !*task->mimetype) {
@@ -341,6 +347,7 @@ extract_task_new (TrackerExtract *extract,
GAsyncResult *res,
GError **error)
{
+ TrackerExtractPrivate *priv = TRACKER_EXTRACT_GET_PRIVATE (extract);
TrackerExtractTask *task;
gchar *mimetype_used;
@@ -377,6 +384,7 @@ extract_task_new (TrackerExtract *extract,
task->file = g_strdup (uri);
task->mimetype = mimetype_used;
task->extract = extract;
+ task->max_text = priv->max_text;
if (task->res) {
GSource *source;
@@ -784,3 +792,12 @@ tracker_extract_file_finish (TrackerExtract *extract,
return g_task_propagate_pointer (G_TASK (res), error);
}
+
+void
+tracker_extract_set_max_text (TrackerExtract *extract,
+ gint max_text)
+{
+ TrackerExtractPrivate *priv = TRACKER_EXTRACT_GET_PRIVATE (extract);
+
+ priv->max_text = max_text;
+}
diff --git a/src/tracker-extract/tracker-extract.h b/src/tracker-extract/tracker-extract.h
index bd88c3f9d..ff573b9d1 100644
--- a/src/tracker-extract/tracker-extract.h
+++ b/src/tracker-extract/tracker-extract.h
@@ -74,6 +74,9 @@ TrackerExtractInfo *
void tracker_extract_dbus_start (TrackerExtract *extract);
void tracker_extract_dbus_stop (TrackerExtract *extract);
+void tracker_extract_set_max_text (TrackerExtract *extract,
+ gint max_text);
+
/* Not DBus API */
void tracker_extract_get_metadata_by_cmdline (TrackerExtract *object,
const gchar *path,
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index 422c53044..74245ce0c 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -40,7 +40,6 @@
#include <libtracker-miners-common/tracker-common.h>
-#include "tracker-config.h"
#include "tracker-main.h"
#include "tracker-extract.h"
#include "tracker-extract-controller.h"
@@ -74,8 +73,6 @@ static gboolean version;
static gchar *domain_ontology_name = NULL;
static guint shutdown_timeout_id = 0;
-static TrackerConfig *config;
-
static GOptionEntry entries[] = {
{ "file", 'f', 0,
G_OPTION_ARG_FILENAME, &filename,
@@ -129,26 +126,8 @@ initialize_priority_and_scheduling (void)
}
}
-static void
-log_option_values (TrackerConfig *config)
-{
-#ifdef G_ENABLE_DEBUG
- if (TRACKER_DEBUG_CHECK (CONFIG)) {
- g_message ("General options:");
- g_message (" Max bytes (per file) ................. %d",
- tracker_config_get_max_bytes (config));
- }
-#endif
-}
-
-TrackerConfig *
-tracker_main_get_config (void)
-{
- return config;
-}
-
static int
-run_standalone (TrackerConfig *config)
+run_standalone (void)
{
TrackerExtract *object;
GFile *file;
@@ -318,20 +297,14 @@ main (int argc, char *argv[])
return EXIT_FAILURE;
}
- config = tracker_config_new ();
-
- /* Extractor command line arguments */
- log_option_values (config);
-
/* Set conditions when we use stand alone settings */
if (filename) {
- return run_standalone (config);
+ return run_standalone ();
}
extract = tracker_extract_new (TRUE, force_module);
if (!extract) {
- g_object_unref (config);
return EXIT_FAILURE;
}
@@ -353,7 +326,6 @@ main (int argc, char *argv[])
if (error) {
g_critical ("Could not start decorator: %s\n", error->message);
- g_object_unref (config);
return EXIT_FAILURE;
}
@@ -362,7 +334,6 @@ main (int argc, char *argv[])
g_critical ("Could not create miner DBus proxy: %s\n", error->message);
g_error_free (error);
g_object_unref (decorator);
- g_object_unref (config);
return EXIT_FAILURE;
}
@@ -431,7 +402,5 @@ main (int argc, char *argv[])
tracker_sparql_connection_close (sparql_connection);
g_object_unref (sparql_connection);
- g_object_unref (config);
-
return EXIT_SUCCESS;
}
diff --git a/src/tracker-extract/tracker-main.h b/src/tracker-extract/tracker-main.h
index 4af2e3e06..5ee12c697 100644
--- a/src/tracker-extract/tracker-main.h
+++ b/src/tracker-extract/tracker-main.h
@@ -21,13 +21,8 @@
#ifndef __TRACKER_MAIN_H__
#define __TRACKER_MAIN_H__
-#include "tracker-config.h"
-
G_BEGIN_DECLS
-/* Enables getting the config object from extractors */
-TrackerConfig *tracker_main_get_config (void);
-
G_END_DECLS
#endif /* __TRACKER_MAIN_H__ */
diff --git a/tests/libtracker-extract/tracker-extract-info-test.c b/tests/libtracker-extract/tracker-extract-info-test.c
index 736350dc1..3f0d9fb94 100644
--- a/tests/libtracker-extract/tracker-extract-info-test.c
+++ b/tests/libtracker-extract/tracker-extract-info-test.c
@@ -29,7 +29,7 @@ test_extract_info_setters (void)
file = g_file_new_for_path ("./imaginary-file-2");
- info = tracker_extract_info_new (file, "imaginary/mime", NULL);
+ info = tracker_extract_info_new (file, "imaginary/mime", NULL, 100);
info_ref = tracker_extract_info_ref (info);
g_assert_true (g_file_equal (file, tracker_extract_info_get_file (info)));
@@ -50,7 +50,7 @@ test_extract_info_empty_objects (void)
file = g_file_new_for_path ("./imaginary-file");
- info = tracker_extract_info_new (file, "imaginary/mime", NULL);
+ info = tracker_extract_info_new (file, "imaginary/mime", NULL, 100);
info_ref = tracker_extract_info_ref (info);
tracker_extract_info_unref (info_ref);
--
GitLab
From f49aa76ec4169572bff55748fa4c3b12f5b89737 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2023 19:18:52 +0200
Subject: [PATCH 6/9] tracker-extract: Avoid file access for persistence
Use a memfd_create() FD, maintained and kept alive by tracker-miner-fs-3.
This FD is obtained through D-Bus, and used for temporary storage. Since
processing of files in the extractor is largely linear nowadays, this
also simplifies the persistent storage to store a single file.
---
config-miners.h.meson.in | 3 +
meson.build | 1 +
src/miners/fs/tracker-files-interface.c | 72 +++++-
.../tracker-extract-controller.c | 65 ++++-
.../tracker-extract-controller.h | 5 +-
.../tracker-extract-decorator.c | 60 ++++-
.../tracker-extract-decorator.h | 10 +-
.../tracker-extract-persistence.c | 241 +++++-------------
.../tracker-extract-persistence.h | 18 +-
src/tracker-extract/tracker-main.c | 9 +-
10 files changed, 278 insertions(+), 206 deletions(-)
diff --git a/config-miners.h.meson.in b/config-miners.h.meson.in
index 666630221..8158e1511 100644
--- a/config-miners.h.meson.in
+++ b/config-miners.h.meson.in
@@ -65,6 +65,9 @@
/* Define to 1 if you have the `strnlen' function. */
#mesondefine HAVE_STRNLEN
+/* Define to 1 if you have the `memfd_create' function. */
+#mesondefine HAVE_MEMFD_CREATE
+
/* Define if we have UPOWER */
#mesondefine HAVE_UPOWER
diff --git a/meson.build b/meson.build
index fb51dea81..46898d9ce 100644
--- a/meson.build
+++ b/meson.build
@@ -378,6 +378,7 @@ conf.set('HAVE_GETLINE', cc.has_function('getline', prefix : '#include <stdio.h>
conf.set('HAVE_POSIX_FADVISE', cc.has_function('posix_fadvise', prefix : '#include <fcntl.h>'))
conf.set('HAVE_STATVFS64', cc.has_header_symbol('sys/statvfs.h', 'statvfs64', args: '-D_LARGEFILE64_SOURCE'))
conf.set('HAVE_STRNLEN', cc.has_function('strnlen', prefix : '#include <string.h>'))
+conf.set('HAVE_MEMFD_CREATE', cc.has_function('memfd_create', prefix : '#define _GNU_SOURCE\n#include <sys/mman.h>'))
conf.set('LOCALEDIR', '"@0@/@1@"'.format(get_option('prefix'), get_option('localedir')))
conf.set('SHAREDIR', '"@0@/@1@"'.format(get_option('prefix'), get_option('datadir')))
diff --git a/src/miners/fs/tracker-files-interface.c b/src/miners/fs/tracker-files-interface.c
index 150e40eab..69c2531ca 100644
--- a/src/miners/fs/tracker-files-interface.c
+++ b/src/miners/fs/tracker-files-interface.c
@@ -23,12 +23,16 @@
#include "tracker-files-interface.h"
+#include <gio/gunixfdlist.h>
+#include <sys/mman.h>
+
struct _TrackerFilesInterface
{
GObject parent_instance;
GDBusConnection *connection;
GSettings *settings;
guint object_id;
+ int fd;
};
enum {
@@ -43,6 +47,9 @@ static const gchar *introspection_xml =
"<node>"
" <interface name='org.freedesktop.Tracker3.Files'>"
" <property name='ExtractorConfig' type='a{sv}' access='read' />"
+ " <method name='GetPersistenceStorage'>"
+ " <arg type='h' direction='out' />"
+ " </method>"
" </interface>"
"</node>";
@@ -53,6 +60,66 @@ tracker_files_interface_init (TrackerFilesInterface *files_interface)
{
}
+static void
+handle_method_call (GDBusConnection *connection,
+ const gchar *sender,
+ const gchar *object_path,
+ const gchar *interface_name,
+ const gchar *method_name,
+ GVariant *parameters,
+ GDBusMethodInvocation *invocation,
+ gpointer user_data)
+{
+ TrackerFilesInterface *files_interface = user_data;
+
+ if (g_strcmp0 (method_name, "GetPersistenceStorage") == 0) {
+ GVariant *out_parameters;
+ g_autoptr (GUnixFDList) fd_list = NULL;
+ g_autoptr (GError) error = NULL;
+ int idx;
+
+ if (files_interface->fd <= 0) {
+#ifdef HAVE_MEMFD_CREATE
+ files_interface->fd = memfd_create ("extract-persistent-storage",
+ MFD_CLOEXEC);
+#else
+ g_autofree gchar *path = NULL;
+
+ path = g_strdup_printf ("%s/tracker-persistence.XXXXXX",
+ g_get_tmp_dir ());
+ files_interface->fd = g_mkstemp_full (path, 0, 0600);
+ unlink (path);
+#endif
+
+ if (files_interface->fd < 0) {
+ g_dbus_method_invocation_return_error (invocation,
+ G_IO_ERROR,
+ G_IO_ERROR_FAILED,
+ "Could not create memfd");
+ return;
+ }
+ }
+
+ fd_list = g_unix_fd_list_new ();
+ idx = g_unix_fd_list_append (fd_list, files_interface->fd, &error);
+
+ if (error) {
+ g_dbus_method_invocation_return_gerror (invocation, error);
+ } else {
+ out_parameters = g_variant_new ("(h)", idx);
+ g_dbus_method_invocation_return_value_with_unix_fd_list (invocation,
+ out_parameters,
+ fd_list);
+ }
+ } else {
+ g_dbus_method_invocation_return_error (invocation,
+ G_DBUS_ERROR,
+ G_DBUS_ERROR_UNKNOWN_METHOD,
+ "Unknown method %s",
+ method_name);
+ }
+}
+
static GVariant *
handle_get_property (GDBusConnection *connection,
const gchar *sender,
@@ -90,7 +157,7 @@ static void
tracker_files_interface_constructed (GObject *object)
{
TrackerFilesInterface *files_interface = TRACKER_FILES_INTERFACE (object);
- GDBusInterfaceVTable vtable = { NULL, handle_get_property, NULL };
+ GDBusInterfaceVTable vtable = { handle_method_call, handle_get_property, NULL };
g_autoptr (GDBusNodeInfo) introspection_data = NULL;
G_OBJECT_CLASS (tracker_files_interface_parent_class)->constructed (object);
@@ -115,6 +182,9 @@ tracker_files_interface_finalize (GObject *object)
g_clear_object (&files_interface->connection);
g_clear_object (&files_interface->settings);
+ if (files_interface->fd)
+ close (files_interface->fd);
+
G_OBJECT_CLASS (tracker_files_interface_parent_class)->finalize (object);
}
diff --git a/src/tracker-extract/tracker-extract-controller.c b/src/tracker-extract/tracker-extract-controller.c
index 404f1f13c..96ceafcab 100644
--- a/src/tracker-extract/tracker-extract-controller.c
+++ b/src/tracker-extract/tracker-extract-controller.c
@@ -23,13 +23,17 @@
#include "tracker-main.h"
+#include <gio/gunixfdlist.h>
+
enum {
PROP_DECORATOR = 1,
PROP_CONNECTION,
+ PROP_PERSISTENCE,
};
struct TrackerExtractControllerPrivate {
TrackerDecorator *decorator;
+ TrackerExtractPersistence *persistence;
GCancellable *cancellable;
GDBusConnection *connection;
GDBusProxy *miner_proxy;
@@ -97,6 +101,38 @@ miner_properties_changed_cb (GDBusProxy *proxy,
update_extract_config (user_data, proxy);
}
+static gboolean
+set_up_persistence (TrackerExtractController *controller,
+ GCancellable *cancellable,
+ GError **error)
+{
+ TrackerExtractControllerPrivate *priv =
+ tracker_extract_controller_get_instance_private (controller);
+ g_autoptr (GUnixFDList) out_fd_list = NULL;
+ g_autoptr (GVariant) variant = NULL;
+ int idx, fd;
+
+ variant = g_dbus_proxy_call_with_unix_fd_list_sync (priv->miner_proxy,
+ "GetPersistenceStorage",
+ NULL,
+ G_DBUS_CALL_FLAGS_NO_AUTO_START,
+ -1,
+ NULL,
+ &out_fd_list,
+ cancellable,
+ error);
+ if (!variant)
+ return FALSE;
+
+ g_variant_get (variant, "(h)", &idx);
+ fd = g_unix_fd_list_get (out_fd_list, idx, error);
+ if (fd < 0)
+ return FALSE;
+
+ tracker_extract_persistence_set_fd (priv->persistence, fd);
+ return TRUE;
+}
+
static void
decorator_raise_error_cb (TrackerDecorator *decorator,
GFile *file,
@@ -173,6 +209,8 @@ tracker_extract_controller_constructed (GObject *object)
G_CALLBACK (miner_properties_changed_cb), object);
update_extract_config (self, self->priv->miner_proxy);
}
+
+ set_up_persistence (self, NULL, NULL);
}
static void
@@ -190,6 +228,12 @@ tracker_extract_controller_get_property (GObject *object,
case PROP_CONNECTION:
g_value_set_object (value, self->priv->connection);
break;
+ case PROP_PERSISTENCE:
+ g_value_set_object (value, self->priv->persistence);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
+ break;
}
}
@@ -209,6 +253,12 @@ tracker_extract_controller_set_property (GObject *object,
case PROP_CONNECTION:
self->priv->connection = g_value_dup_object (value);
break;
+ case PROP_PERSISTENCE:
+ self->priv->persistence = g_value_dup_object (value);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
+ break;
}
}
@@ -223,6 +273,7 @@ tracker_extract_controller_dispose (GObject *object)
}
g_clear_object (&self->priv->decorator);
+ g_clear_object (&self->priv->persistence);
G_OBJECT_CLASS (tracker_extract_controller_parent_class)->dispose (object);
}
@@ -255,6 +306,14 @@ tracker_extract_controller_class_init (TrackerExtractControllerClass *klass)
G_PARAM_STATIC_STRINGS |
G_PARAM_READWRITE |
G_PARAM_CONSTRUCT_ONLY));
+ g_object_class_install_property (object_class,
+ PROP_PERSISTENCE,
+ g_param_spec_object ("persistence",
+ NULL, NULL,
+ TRACKER_TYPE_EXTRACT_PERSISTENCE,
+ G_PARAM_STATIC_STRINGS |
+ G_PARAM_READWRITE |
+ G_PARAM_CONSTRUCT_ONLY));
}
static void
@@ -264,13 +323,15 @@ tracker_extract_controller_init (TrackerExtractController *self)
}
TrackerExtractController *
-tracker_extract_controller_new (TrackerDecorator *decorator,
- GDBusConnection *connection)
+tracker_extract_controller_new (TrackerDecorator *decorator,
+ GDBusConnection *connection,
+ TrackerExtractPersistence *persistence)
{
g_return_val_if_fail (TRACKER_IS_DECORATOR (decorator), NULL);
return g_object_new (TRACKER_TYPE_EXTRACT_CONTROLLER,
"decorator", decorator,
"connection", connection,
+ "persistence", persistence,
NULL);
}
diff --git a/src/tracker-extract/tracker-extract-controller.h b/src/tracker-extract/tracker-extract-controller.h
index 7d8a70816..3ba85751c 100644
--- a/src/tracker-extract/tracker-extract-controller.h
+++ b/src/tracker-extract/tracker-extract-controller.h
@@ -47,8 +47,9 @@ struct TrackerExtractControllerClass {
};
GType tracker_extract_controller_get_type (void) G_GNUC_CONST;
-TrackerExtractController * tracker_extract_controller_new (TrackerDecorator *decorator,
- GDBusConnection *connection);
+TrackerExtractController * tracker_extract_controller_new (TrackerDecorator *decorator,
+ GDBusConnection *connection,
+ TrackerExtractPersistence *persistence);
G_END_DECLS
diff --git a/src/tracker-extract/tracker-extract-decorator.c b/src/tracker-extract/tracker-extract-decorator.c
index b5505a466..2783d3fd0 100644
--- a/src/tracker-extract/tracker-extract-decorator.c
+++ b/src/tracker-extract/tracker-extract-decorator.c
@@ -26,7 +26,9 @@
#include "tracker-extract-persistence.h"
enum {
- PROP_EXTRACTOR = 1
+ PROP_0,
+ PROP_EXTRACTOR,
+ PROP_PERSISTENCE,
};
#define MAX_EXTRACTING_FILES 1
@@ -87,6 +89,12 @@ tracker_extract_decorator_get_property (GObject *object,
case PROP_EXTRACTOR:
g_value_set_object (value, priv->extractor);
break;
+ case PROP_PERSISTENCE:
+ g_value_set_object (value, priv->persistence);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
+ break;
}
}
@@ -104,6 +112,12 @@ tracker_extract_decorator_set_property (GObject *object,
case PROP_EXTRACTOR:
priv->extractor = g_value_dup_object (value);
break;
+ case PROP_PERSISTENCE:
+ priv->persistence = g_value_dup_object (value);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
+ break;
}
}
@@ -121,6 +135,7 @@ tracker_extract_decorator_finalize (GObject *object)
g_timer_destroy (priv->timer);
g_clear_object (&priv->index_proxy);
+ g_clear_object (&priv->persistence);
G_OBJECT_CLASS (tracker_extract_decorator_parent_class)->finalize (object);
}
@@ -163,7 +178,7 @@ get_metadata_cb (TrackerExtract *extract,
priv = tracker_extract_decorator_get_instance_private (TRACKER_EXTRACT_DECORATOR (data->decorator));
info = tracker_extract_file_finish (extract, result, &error);
- tracker_extract_persistence_remove_file (priv->persistence, data->file);
+ tracker_extract_persistence_set_file (priv->persistence, NULL);
if (data->cancellable && data->signal_id != 0) {
g_cancellable_disconnect (data->cancellable, data->signal_id);
@@ -232,7 +247,7 @@ task_cancellable_cancelled_cb (GCancellable *cancellable,
* this as a failed operation.
*/
priv = tracker_extract_decorator_get_instance_private (TRACKER_EXTRACT_DECORATOR (data->decorator));
- tracker_extract_persistence_remove_file (priv->persistence, data->file);
+ tracker_extract_persistence_set_file (priv->persistence, NULL);
uri = g_file_get_uri (data->file);
g_debug ("Cancelled task for '%s' was currently being "
@@ -303,7 +318,7 @@ decorator_next_item_cb (TrackerDecorator *decorator,
g_debug ("Extracting metadata for '%s'", tracker_decorator_info_get_url (info));
- tracker_extract_persistence_add_file (priv->persistence, data->file);
+ tracker_extract_persistence_set_file (priv->persistence, data->file);
g_set_object (&data->cancellable, g_task_get_cancellable (task));
@@ -366,6 +381,22 @@ tracker_extract_decorator_resumed (TrackerMiner *miner)
decorator_get_next_file (TRACKER_DECORATOR (miner));
}
+static void
+tracker_extract_decorator_started (TrackerMiner *miner)
+{
+ TrackerExtractDecorator *decorator = TRACKER_EXTRACT_DECORATOR (miner);
+ TrackerExtractDecoratorPrivate *priv =
+ tracker_extract_decorator_get_instance_private (decorator);
+ GFile *file;
+
+ file = tracker_extract_persistence_get_file (priv->persistence);
+
+ if (file)
+ decorator_ignore_file (file, decorator, "Crash/hang handling file", NULL);
+
+ TRACKER_MINER_CLASS (tracker_extract_decorator_parent_class)->started (miner);
+}
+
static void
tracker_extract_decorator_items_available (TrackerDecorator *decorator)
{
@@ -426,6 +457,7 @@ tracker_extract_decorator_class_init (TrackerExtractDecoratorClass *klass)
miner_class->paused = tracker_extract_decorator_paused;
miner_class->resumed = tracker_extract_decorator_resumed;
+ miner_class->started = tracker_extract_decorator_started;
decorator_class->items_available = tracker_extract_decorator_items_available;
decorator_class->finished = tracker_extract_decorator_finished;
@@ -440,6 +472,14 @@ tracker_extract_decorator_class_init (TrackerExtractDecoratorClass *klass)
G_PARAM_READWRITE |
G_PARAM_CONSTRUCT_ONLY |
G_PARAM_STATIC_STRINGS));
+ g_object_class_install_property (object_class,
+ PROP_PERSISTENCE,
+ g_param_spec_object ("persistence",
+ NULL, NULL,
+ TRACKER_TYPE_EXTRACT_PERSISTENCE,
+ G_PARAM_READWRITE |
+ G_PARAM_CONSTRUCT_ONLY |
+ G_PARAM_STATIC_STRINGS));
}
static void
@@ -585,8 +625,6 @@ tracker_extract_decorator_initable_init (GInitable *initable,
ret = FALSE;
}
- priv->persistence = tracker_extract_persistence_initialize (persistence_ignore_file,
- decorator);
out:
g_clear_object (&conn);
@@ -601,14 +639,16 @@ tracker_extract_decorator_initable_iface_init (GInitableIface *iface)
}
TrackerDecorator *
-tracker_extract_decorator_new (TrackerSparqlConnection *connection,
- TrackerExtract *extract,
- GCancellable *cancellable,
- GError **error)
+tracker_extract_decorator_new (TrackerSparqlConnection *connection,
+ TrackerExtract *extract,
+ TrackerExtractPersistence *persistence,
+ GCancellable *cancellable,
+ GError **error)
{
return g_initable_new (TRACKER_TYPE_EXTRACT_DECORATOR,
cancellable, error,
"connection", connection,
"extractor", extract,
+ "persistence", persistence,
NULL);
}
diff --git a/src/tracker-extract/tracker-extract-decorator.h b/src/tracker-extract/tracker-extract-decorator.h
index 8cbf74891..b77d4fb62 100644
--- a/src/tracker-extract/tracker-extract-decorator.h
+++ b/src/tracker-extract/tracker-extract-decorator.h
@@ -24,6 +24,7 @@
#include <libtracker-miner/tracker-miner.h>
#include "tracker-extract.h"
+#include "tracker-extract-persistence.h"
G_BEGIN_DECLS
@@ -47,10 +48,11 @@ struct TrackerExtractDecoratorClass {
GType tracker_extract_decorator_get_type (void) G_GNUC_CONST;
-TrackerDecorator * tracker_extract_decorator_new (TrackerSparqlConnection *connection,
- TrackerExtract *extractor,
- GCancellable *cancellable,
- GError **error);
+TrackerDecorator * tracker_extract_decorator_new (TrackerSparqlConnection *connection,
+ TrackerExtract *extractor,
+ TrackerExtractPersistence *persistence,
+ GCancellable *cancellable,
+ GError **error);
G_END_DECLS
diff --git a/src/tracker-extract/tracker-extract-persistence.c b/src/tracker-extract/tracker-extract-persistence.c
index bf356bc4f..995be2596 100644
--- a/src/tracker-extract/tracker-extract-persistence.c
+++ b/src/tracker-extract/tracker-extract-persistence.c
@@ -23,217 +23,108 @@ typedef struct _TrackerExtractPersistencePrivate TrackerExtractPersistencePrivat
struct _TrackerExtractPersistencePrivate
{
- GFile *tmp_dir;
+ int fd;
};
G_DEFINE_TYPE_WITH_PRIVATE (TrackerExtractPersistence, tracker_extract_persistence, G_TYPE_OBJECT)
static void
-tracker_extract_persistence_class_init (TrackerExtractPersistenceClass *klass)
+tracker_extract_persistence_finalize (GObject *object)
{
+ TrackerExtractPersistence *persistence =
+ TRACKER_EXTRACT_PERSISTENCE (object);
+ TrackerExtractPersistencePrivate *priv =
+ tracker_extract_persistence_get_instance_private (persistence);
+
+ if (priv->fd > 0)
+ close (priv->fd);
+
+ G_OBJECT_CLASS (tracker_extract_persistence_parent_class)->finalize (object);
}
static void
-tracker_extract_persistence_init (TrackerExtractPersistence *persistence)
+tracker_extract_persistence_class_init (TrackerExtractPersistenceClass *klass)
{
- TrackerExtractPersistencePrivate *priv;
- gchar *dirname, *tmp_path;
-
- priv = tracker_extract_persistence_get_instance_private (persistence);
-
- dirname = g_strdup_printf ("tracker-extract-3-files.%d", getuid ());
- tmp_path = g_build_filename (g_get_tmp_dir (), dirname, NULL);
- g_free (dirname);
+ GObjectClass *object_class = G_OBJECT_CLASS (klass);
- if (g_mkdir_with_parents (tmp_path, 0700) != 0) {
- g_critical ("The directory %s could not be created, or has the wrong permissions",
- tmp_path);
- g_assert_not_reached ();
- }
-
- priv->tmp_dir = g_file_new_for_path (tmp_path);
- g_free (tmp_path);
+ object_class->finalize = tracker_extract_persistence_finalize;
}
-static GFile *
-persistence_create_symlink_file (TrackerExtractPersistence *persistence,
- GFile *file)
+static void
+tracker_extract_persistence_init (TrackerExtractPersistence *persistence)
{
- TrackerExtractPersistencePrivate *priv;
- gchar *path, *md5;
- GFile *link_file;
-
- priv = tracker_extract_persistence_get_instance_private (persistence);
- path = g_file_get_path (file);
- md5 = g_compute_checksum_for_string (G_CHECKSUM_MD5, path, -1);
- link_file = g_file_get_child (priv->tmp_dir, md5);
-
- g_free (path);
- g_free (md5);
-
- return link_file;
}
-static GFile *
-persistence_symlink_get_file (GFileInfo *info)
+TrackerExtractPersistence *
+tracker_extract_persistence_new (void)
{
- const gchar *symlink_name, *symlink_target;
- gchar *md5;
- GFile *file = NULL;
-
- symlink_name = g_file_info_get_name (info);
- symlink_target = g_file_info_get_symlink_target (info);
-
- if (!g_path_is_absolute (symlink_target)) {
- g_critical ("Symlink paths must be absolute, '%s' points to '%s'",
- symlink_name, symlink_target);
- return NULL;
- }
-
- md5 = g_compute_checksum_for_string (G_CHECKSUM_MD5, symlink_target, -1);
-
- if (g_strcmp0 (symlink_name, md5) == 0) {
- file = g_file_new_for_path (symlink_target);
- } else {
- g_critical ("path MD5 for '%s' doesn't match with symlink '%s'",
- symlink_target, symlink_name);
- }
-
- g_free (md5);
-
- return file;
+ return g_object_new (TRACKER_TYPE_EXTRACT_PERSISTENCE,
+ NULL);
}
-static gboolean
-persistence_store_file (TrackerExtractPersistence *persistence,
- GFile *file)
+void
+tracker_extract_persistence_set_fd (TrackerExtractPersistence *persistence,
+ int fd)
{
- GError *error = NULL;
- gboolean success;
- GFile *link_file;
- gchar *path;
-
- path = g_file_get_path (file);
- link_file = persistence_create_symlink_file (persistence, file);
+ TrackerExtractPersistencePrivate *priv =
+ tracker_extract_persistence_get_instance_private (persistence);
- success = g_file_make_symbolic_link (link_file, path, NULL, &error);
-
- if (!success) {
- g_warning ("Could not save '%s' into failsafe persistence store: %s",
- path, error ? error->message : "no error given");
- g_clear_error (&error);
- }
-
- g_object_unref (link_file);
- g_free (path);
-
- return success;
+ if (priv->fd > 0)
+ close (priv->fd);
+ priv->fd = fd;
}
-static gboolean
-persistence_remove_file (TrackerExtractPersistence *persistence,
- GFile *file)
+void
+tracker_extract_persistence_set_file (TrackerExtractPersistence *persistence,
+ GFile *file)
{
- GError *error = NULL;
- GFile *link_file;
- gboolean success;
-
- link_file = persistence_create_symlink_file (persistence, file);
- success = g_file_delete (link_file, NULL, &error);
+ TrackerExtractPersistencePrivate *priv =
+ tracker_extract_persistence_get_instance_private (persistence);
+ g_autofree gchar *path = NULL;
+ int len, written = 0, retval;
- if (!success) {
- gchar *path = g_file_get_path (file);
+ g_return_if_fail (TRACKER_IS_EXTRACT_PERSISTENCE (persistence));
+ g_return_if_fail (!file || G_IS_FILE (file));
- g_warning ("Could not delete '%s' from failsafe persistence store",
- path);
- g_free (path);
+ if (file) {
+ path = g_file_get_path (file);
+ } else {
+ path = g_strdup ("");
}
- g_object_unref (link_file);
+ /* Write also the trailing \0 */
+ len = strlen (path) + 1;
- return success;
-}
+ lseek (priv->fd, 0, SEEK_SET);
-static void
-persistence_retrieve_files (TrackerExtractPersistence *persistence,
- TrackerFileRecoveryFunc ignore_func,
- gpointer user_data)
-{
- TrackerExtractPersistencePrivate *priv;
- GFileEnumerator *enumerator;
- GFileInfo *info;
-
- priv = tracker_extract_persistence_get_instance_private (persistence);
- enumerator = g_file_enumerate_children (priv->tmp_dir,
- G_FILE_ATTRIBUTE_STANDARD_NAME ","
- G_FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- NULL, NULL);
- if (!enumerator)
- return;
-
- while ((info = g_file_enumerator_next_file (enumerator, NULL, NULL)) != NULL) {
- GFile *file, *symlink_file;
-
- symlink_file = g_file_enumerator_get_child (enumerator, info);
- file = persistence_symlink_get_file (info);
-
- if (!file) {
- /* If we got here, persistence_symlink_get_file() already emitted a g_critical */
- g_object_unref (symlink_file);
- g_object_unref (info);
- continue;
- }
-
- /* Delete the symlink.
- */
- g_file_delete (symlink_file, NULL, NULL);
- g_object_unref (symlink_file);
-
- /* Trigger ignore func for the symlink target */
- ignore_func (file, user_data);
-
- g_object_unref (file);
- g_object_unref (info);
- }
+ while (TRUE) {
+ retval = write (priv->fd, &path[written], len - written);
+ if (retval < 0)
+ break;
- g_file_enumerator_close (enumerator, NULL, NULL);
- g_object_unref (enumerator);
-}
-
-TrackerExtractPersistence *
-tracker_extract_persistence_initialize (TrackerFileRecoveryFunc ignore_func,
- gpointer user_data)
-{
- static TrackerExtractPersistence *persistence = NULL;
-
- if (!persistence) {
- persistence = g_object_new (TRACKER_TYPE_EXTRACT_PERSISTENCE,
- NULL);
- persistence_retrieve_files (persistence,
- ignore_func,
- user_data);
+ written += retval;
+ if (written >= len)
+ break;
}
-
- return persistence;
}
-void
-tracker_extract_persistence_add_file (TrackerExtractPersistence *persistence,
- GFile *file)
+GFile *
+tracker_extract_persistence_get_file (TrackerExtractPersistence *persistence)
{
- g_return_if_fail (TRACKER_IS_EXTRACT_PERSISTENCE (persistence));
- g_return_if_fail (G_IS_FILE (file));
+ TrackerExtractPersistencePrivate *priv =
+ tracker_extract_persistence_get_instance_private (persistence);
+ gchar buf[2048];
+ int len;
- persistence_store_file (persistence, file);
-}
+ g_return_val_if_fail (TRACKER_IS_EXTRACT_PERSISTENCE (persistence), NULL);
-void
-tracker_extract_persistence_remove_file (TrackerExtractPersistence *persistence,
- GFile *file)
-{
- g_return_if_fail (TRACKER_IS_EXTRACT_PERSISTENCE (persistence));
- g_return_if_fail (G_IS_FILE (file));
+ lseek (priv->fd, 0, SEEK_SET);
+ len = read (priv->fd, buf, sizeof (buf));
+ if (len <= 0)
+ return NULL;
+ if (buf[0] == '\0')
+ return NULL;
- persistence_remove_file (persistence, file);
+ buf[len - 1] = '\0';
+ return g_file_new_for_path (buf);
}
diff --git a/src/tracker-extract/tracker-extract-persistence.h b/src/tracker-extract/tracker-extract-persistence.h
index b935b0898..c1d498ef7 100644
--- a/src/tracker-extract/tracker-extract-persistence.h
+++ b/src/tracker-extract/tracker-extract-persistence.h
@@ -34,9 +34,6 @@ G_BEGIN_DECLS
typedef struct _TrackerExtractPersistence TrackerExtractPersistence;
typedef struct _TrackerExtractPersistenceClass TrackerExtractPersistenceClass;
-typedef void (* TrackerFileRecoveryFunc) (GFile *file,
- gpointer user_data);
-
struct _TrackerExtractPersistence
{
GObject parent_instance;
@@ -49,14 +46,15 @@ struct _TrackerExtractPersistenceClass
GType tracker_extract_persistence_get_type (void) G_GNUC_CONST;
-TrackerExtractPersistence *
- tracker_extract_persistence_initialize (TrackerFileRecoveryFunc ignore_func,
- gpointer user_data);
+TrackerExtractPersistence * tracker_extract_persistence_new (void);
+
+void tracker_extract_persistence_set_fd (TrackerExtractPersistence *persistence,
+ int fd);
+
+GFile * tracker_extract_persistence_get_file (TrackerExtractPersistence *persistence);
-void tracker_extract_persistence_add_file (TrackerExtractPersistence *persistence,
- GFile *file);
-void tracker_extract_persistence_remove_file (TrackerExtractPersistence *persistence,
- GFile *file);
+void tracker_extract_persistence_set_file (TrackerExtractPersistence *persistence,
+ GFile *file);
G_END_DECLS
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index 74245ce0c..90527038a 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -44,6 +44,7 @@
#include "tracker-extract.h"
#include "tracker-extract-controller.h"
#include "tracker-extract-decorator.h"
+#include "tracker-extract-persistence.h"
#ifdef THREAD_ENABLE_TRACE
#warning Main thread traces enabled
@@ -232,6 +233,7 @@ main (int argc, char *argv[])
GMainLoop *my_main_loop;
GDBusConnection *connection;
TrackerMinerProxy *proxy;
+ TrackerExtractPersistence *persistence;
TrackerSparqlConnection *sparql_connection;
TrackerDomainOntology *domain_ontology;
gchar *dbus_name, *miner_dbus_name;
@@ -322,7 +324,9 @@ main (int argc, char *argv[])
return EXIT_FAILURE;
}
- decorator = tracker_extract_decorator_new (sparql_connection, extract, NULL, &error);
+ persistence = tracker_extract_persistence_new ();
+
+ decorator = tracker_extract_decorator_new (sparql_connection, extract, persistence, NULL, &error);
if (error) {
g_critical ("Could not start decorator: %s\n", error->message);
@@ -344,7 +348,7 @@ main (int argc, char *argv[])
tracker_locale_sanity_check ();
- controller = tracker_extract_controller_new (decorator, connection);
+ controller = tracker_extract_controller_new (decorator, connection, persistence);
/* Request DBus name */
dbus_name = tracker_domain_ontology_get_domain (domain_ontology, DBUS_NAME_SUFFIX);
@@ -396,6 +400,7 @@ main (int argc, char *argv[])
g_object_unref (extract);
g_object_unref (decorator);
g_object_unref (controller);
+ g_object_unref (persistence);
g_object_unref (proxy);
g_object_unref (connection);
tracker_domain_ontology_unref (domain_ontology);
--
GitLab
From 756650244fad90f970c422b413bce827b92b1a41 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2023 23:27:56 +0200
Subject: [PATCH 7/9] tracker-extract: Disable GstRegistry forking
This is going nowhere with the sandbox. Also disable some more
needless GST plugins.
---
src/tracker-extract/tracker-extract-gstreamer.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/tracker-extract/tracker-extract-gstreamer.c b/src/tracker-extract/tracker-extract-gstreamer.c
index ea1985aa2..7399a6ff1 100644
--- a/src/tracker-extract/tracker-extract-gstreamer.c
+++ b/src/tracker-extract/tracker-extract-gstreamer.c
@@ -1410,7 +1410,9 @@ tracker_extract_module_init (GError **error)
/* Lifted from totem-video-thumbnailer */
const gchar *blocklisted[] = {
"bcmdec",
- "fluiddec",
+ "camerabin",
+ "fluidsynthmidi",
+ "libcamera",
"vaapi",
"video4linux2",
"nvmpegvideodec",
@@ -1427,6 +1429,7 @@ tracker_extract_module_init (GError **error)
GstRegistry *registry;
guint i;
+ gst_registry_fork_set_enabled (FALSE);
gst_init (NULL, NULL);
registry = gst_registry_get ();
--
GitLab
From 868d5eba24daa291c4b24a5b574e246ac5538b66 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Fri, 22 Sep 2023 23:26:38 +0200
Subject: [PATCH 8/9] libtracker-miners-common: Extend seccomp rules
The plan is to extend the seccomp jail so it affects the full
tracker-extract-3 process. With the changes in the previous
commits we've removed the need for filesystem write access.
We have some remaining outliers, that we're largely sorting
out with rules to error out softly (instead of through SIGSYS).
The only new allowed syscalls are fstatfs and prlimit64 with a
NULL new_limit struct.
---
.../tracker-seccomp.c | 20 +++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/src/libtracker-miners-common/tracker-seccomp.c b/src/libtracker-miners-common/tracker-seccomp.c
index 1768ef1a1..620158d44 100644
--- a/src/libtracker-miners-common/tracker-seccomp.c
+++ b/src/libtracker-miners-common/tracker-seccomp.c
@@ -140,6 +140,7 @@ tracker_seccomp_init (void)
ALLOW_RULE (lstat);
ALLOW_RULE (lstat64);
ALLOW_RULE (statx);
+ ALLOW_RULE (fstatfs);
ALLOW_RULE (access);
ALLOW_RULE (faccessat);
ALLOW_RULE (faccessat2);
@@ -218,6 +219,22 @@ tracker_seccomp_init (void)
ALLOW_RULE (getpeername);
ALLOW_RULE (shutdown);
+ ERROR_RULE (inotify_init1, EINVAL);
+ ERROR_RULE (inotify_init, EINVAL);
+
+ ERROR_RULE (mkdir, EPERM);
+ ERROR_RULE (rename, EPERM);
+ ERROR_RULE (unlink, EPERM);
+ ERROR_RULE (ioctl, EBADF);
+ ERROR_RULE (bind, EACCES);
+ ERROR_RULE (setsockopt, EBADF);
+ ERROR_RULE (sched_getattr, EPERM);
+
+ /* Allow prlimit64, only if no new limits are being set */
+ if (seccomp_rule_add (ctx, SCMP_ACT_ALLOW, SCMP_SYS(prlimit64), 1,
+ SCMP_CMP(2, SCMP_CMP_EQ, 0)) < 0)
+ goto out;
+
/* Special requirements for socket/socketpair, only on AF_UNIX/AF_LOCAL */
if (seccomp_rule_add (ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), 1,
SCMP_CMP(0, SCMP_CMP_EQ, AF_UNIX)) < 0)
@@ -225,6 +242,9 @@ tracker_seccomp_init (void)
if (seccomp_rule_add (ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), 1,
SCMP_CMP(0, SCMP_CMP_EQ, AF_LOCAL)) < 0)
goto out;
+ if (seccomp_rule_add (ctx, SCMP_ACT_ERRNO (EACCES), SCMP_SYS(socket), 1,
+ SCMP_CMP(0, SCMP_CMP_EQ, AF_NETLINK)) < 0)
+ goto out;
if (seccomp_rule_add (ctx, SCMP_ACT_ALLOW, SCMP_SYS(socketpair), 1,
SCMP_CMP(0, SCMP_CMP_EQ, AF_UNIX)) < 0)
goto out;
--
GitLab
From 1513d41ef137d1083309b2f096a7794d25496f9f Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Fri, 22 Sep 2023 23:14:38 +0200
Subject: [PATCH 9/9] tracker-extract: Extend seccomp jail to full process
Currently, our main thread is exempted from the seccomp jail.
This was so we could do some menial tasks (e.g. persistence handling
to recover from runtime errors, or error reports on failed extraction)
without caring much about plugging seccomp holes.
It may be preferable to extend the seccomp jail to the full process
instead, so do that. Now the only thing happening prior to setting
up the seccomp jail is the setting up of nice/scheduler/ioprio
priorities. Everything else, and every thread spawned afterwards is
covered by seccomp.
Related: https://gitlab.gnome.org/GNOME/tracker-miners/-/issues/277
---
src/tracker-extract/tracker-extract.c | 8 --------
src/tracker-extract/tracker-main.c | 24 ++++++++++++++++--------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/src/tracker-extract/tracker-extract.c b/src/tracker-extract/tracker-extract.c
index 749e73826..994c00c83 100644
--- a/src/tracker-extract/tracker-extract.c
+++ b/src/tracker-extract/tracker-extract.c
@@ -30,8 +30,6 @@
#include <gio/gunixinputstream.h>
#include <gio/gunixfdlist.h>
-#include <libtracker-miners-common/tracker-common.h>
-
#include <libtracker-extract/tracker-extract.h>
#include "tracker-extract.h"
@@ -537,9 +535,6 @@ get_metadata (TrackerExtractTask *task)
static gpointer
single_thread_get_metadata (GAsyncQueue *queue)
{
- if (!tracker_seccomp_init ())
- g_assert_not_reached ();
-
while (TRUE) {
TrackerExtractTask *task;
@@ -711,9 +706,6 @@ tracker_extract_get_metadata_by_cmdline (TrackerExtract *object,
NULL,
&task->func);
- if (!tracker_seccomp_init ())
- g_assert_not_reached ();
-
if (!filter_module (object, task->module) &&
get_file_metadata (task, &info, NULL)) {
resource = tracker_extract_info_get_resource (info);
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index 90527038a..b5cf53fee 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -141,9 +141,6 @@ run_standalone (void)
output_format_name = "turtle";
}
- /* This makes sure we don't steal all the system's resources */
- initialize_priority_and_scheduling ();
-
/* Look up the output format by name */
enum_class = g_type_class_ref (TRACKER_TYPE_SERIALIZATION_FORMAT);
enum_value = g_enum_get_value_by_nick (enum_class, output_format_name);
@@ -222,8 +219,8 @@ on_decorator_finished (TrackerDecorator *decorator,
main_loop);
}
-int
-main (int argc, char *argv[])
+static int
+do_main (int argc, char *argv[])
{
GOptionContext *context;
GError *error = NULL;
@@ -242,9 +239,6 @@ main (int argc, char *argv[])
bind_textdomain_codeset (GETTEXT_PACKAGE, "UTF-8");
textdomain (GETTEXT_PACKAGE);
- /* This makes sure we don't steal all the system's resources */
- initialize_priority_and_scheduling ();
-
/* Translators: this message will appear immediately after the */
/* usage string - Usage: COMMAND [OPTION]... <THIS_MESSAGE> */
context = g_option_context_new (_("— Extract file meta data"));
@@ -409,3 +403,17 @@ main (int argc, char *argv[])
return EXIT_SUCCESS;
}
+
+int
+main (int argc, char *argv[])
+{
+ /* This function is untouchable! Add things to do_main() */
+
+ /* This makes sure we don't steal all the system's resources */
+ initialize_priority_and_scheduling ();
+
+ if (!tracker_seccomp_init ())
+ g_assert_not_reached ();
+
+ return do_main (argc, argv);
+}
--
GitLab
