[users] Add extensible field serialization to export_users command #497

Refactored export_users to support extensible field definitions.

Fields can now be defined as dictionaries to enable:
- custom serialization via callables
- extraction of related objects using "fields"
- traversal of relations via dot notation

Replaced hardcoded "organizations" handling with a callable-based
implementation.

Added support for "prefetch_related" to optimize queries and avoid N+1 issues.

Updated tests and documentation to reflect the new configuration format.

Closes #497

Author: pandafy

docs/user/settings.rst

@@ -81,7 +81,7 @@ without having to specify the international prefix.
 ``OPENWISP_USERS_EXPORT_USERS_COMMAND_CONFIG``
 ----------------------------------------------
 
-============ =============================
+============ ==========================================================================
 **type**:    ``dict``
 **default**: .. code-block:: python
 
@@ -101,17 +101,101 @@ without having to specify the international prefix.
                          "location",
                          "notes",
                          "language",
-                         "organizations",
+                         {
+                             "name": "organizations",
+                             "callable": openwisp_users.settings._export_organizations,
+                         },
                      ],
                      "select_related": [],
+                     "prefetch_related": [],
                  }
-============ =============================
+============ ==========================================================================
 
-This setting can be used to configure the exported fields for the
-:ref:`export_users` command.
+This setting configures the fields exported by the :ref:`export_users`
+management command.
 
-The ``select_related`` property can be used to optimize the database
-query.
+Field definitions
+~~~~~~~~~~~~~~~~~
+
+Each entry in ``fields`` can be either:
+
+- A string, representing a direct attribute of the user model:
+
+  .. code-block:: python
+
+      "email"
+
+- A dictionary, allowing advanced customization:
+
+  .. code-block:: python
+
+      {
+          "name": "organizations",
+          "callable": my_custom_function,
+      }
+
+The following keys are supported in field dictionaries:
+
+- ``name`` (str): the field name used as the CSV column header.
+- ``callable`` (callable, optional): a function that takes the user
+  instance as input and returns the value to be exported.
+- ``fields`` (list of str, optional): a list of attributes to extract from
+  a related object or queryset.
+
+Priority order:
+
+- If ``callable`` is provided, it is used.
+- Else if ``fields`` is provided, the related object(s) are serialized.
+- Otherwise, the value is resolved using ``name``.
+
+Related objects
+~~~~~~~~~~~~~~~
+
+You can export fields from related models in two ways:
+
+- **Dot notation** (for ``ForeignKey`` or ``OneToOne``):
+
+  .. code-block:: python
+
+      "profile.phone_number"
+
+- **Structured extraction using ``fields``**:
+
+  .. code-block:: python
+
+      {
+          "name": "groups",
+          "fields": ["name"],
+      }
+
+If the attribute resolves to a queryset (e.g. reverse ``ForeignKey`` or
+``ManyToMany``), multiple values are serialized into a single CSV cell.
+
+Query optimization
+~~~~~~~~~~~~~~~~~~
+
+- ``select_related`` can be used for ``ForeignKey`` and ``OneToOne``
+  relations.
+- ``prefetch_related`` can be used for reverse relations and
+  ``ManyToMany`` fields.
+
+Example:
+
+.. code-block:: python
+
+    {
+        "fields": [
+            "username",
+            "email",
+            "profile.phone_number",
+            {
+                "name": "groups",
+                "fields": ["name"],
+            },
+        ],
+        "select_related": ["profile"],
+        "prefetch_related": ["groups"],
+    }
 
 .. _openwisp_users_user_password_expiration:
 

openwisp_users/management/commands/export_users.py

@@ -9,6 +9,35 @@
 User = get_user_model()
 
 
+def normalize_field(field):
+    """Normalize a string or dict field definition to a dict."""
+    if isinstance(field, dict):
+        return field
+    return {"name": field}
+
+
+def resolve_attr(obj, attr_name):
+    """Return the attribute value on obj, or None if it does not exist."""
+    try:
+        return getattr(obj, attr_name)
+    except ObjectDoesNotExist:
+        return None
+
+
+def serialize_related(manager, subfields):
+    """Serialize a RelatedManager queryset using the given subfields.
+
+    Single subfield → comma-separated values: val1,val2,...
+    Multiple subfields → tuple-per-row format: ((v1,v2),(v3,v4))
+    """
+    rows = [[str(getattr(obj, f, "")) for f in subfields] for obj in manager.iterator()]
+    if not rows:
+        return ""
+    if len(subfields) == 1:
+        return ",".join(row[0] for row in rows)
+    return "(" + ",".join("(" + ",".join(row) + ")" for row in rows) + ")"
+
+
 class Command(BaseCommand):
     help = "Exports user data to a CSV file"
 
@@ -33,47 +62,72 @@ def handle(self, *args, **options):
         fields = app_settings.EXPORT_USERS_COMMAND_CONFIG.get("fields", []).copy()
         # Get the fields to be excluded from the command-line argument
         exclude_fields = options.get("exclude_fields").split(",")
-        # Remove excluded fields from the export fields
-        fields = [field for field in fields if field not in exclude_fields]
-        # Fetch all user data in a single query using select_related for related models
-        queryset = User.objects.select_related(
-            *app_settings.EXPORT_USERS_COMMAND_CONFIG.get("select_related", []),
-        ).order_by("date_joined")
+        # Remove excluded fields from the export fields (match on the field name)
+        fields = [
+            field
+            for field in fields
+            if normalize_field(field)["name"] not in exclude_fields
+        ]
+        # Fetch all user data using select_related and prefetch_related
+        queryset = (
+            User.objects.select_related(
+                *app_settings.EXPORT_USERS_COMMAND_CONFIG.get("select_related", []),
+            )
+            .prefetch_related(
+                *app_settings.EXPORT_USERS_COMMAND_CONFIG.get("prefetch_related", []),
+            )
+            .order_by("date_joined")
+        )
 
         # Prepare a CSV writer
         filename = options.get("filename")
-        csv_file = open(filename, "w", newline="")
-        csv_writer = csv.writer(csv_file)
-
-        # Write header row
-        csv_writer.writerow(fields)
-
-        # Write data rows
-        for user in queryset.iterator():
-            data_row = []
-            for field in fields:
-                # Extract the value from related models
-                if "." in field:
-                    related_model, related_field = field.split(".")
-                    try:
-                        related_value = getattr(
-                            getattr(user, related_model), related_field
-                        )
-                    except ObjectDoesNotExist:
-                        data_row.append("")
-                    else:
-                        data_row.append(related_value)
-                elif field == "organizations":
-                    organizations = []
-                    for org_id, user_perm in user.organizations_dict.items():
-                        organizations.append(f'({org_id},{user_perm["is_admin"]})')
-                    data_row.append("\n".join(organizations))
-                else:
-                    data_row.append(getattr(user, field))
-            csv_writer.writerow(data_row)
-
-        # Close the CSV file
-        csv_file.close()
+        with open(filename, "w", newline="") as csv_file:
+            csv_writer = csv.writer(csv_file)
+            # Write header row using the name of each field
+            csv_writer.writerow([normalize_field(f)["name"] for f in fields])
+
+            # Write data rows
+            for user in queryset.iterator():
+                csv_writer.writerow(
+                    [self._get_field_value(user, field) for field in fields]
+                )
         self.stdout.write(
             self.style.SUCCESS(f"User data exported successfully to {filename}!")
         )
+
+    def _get_field_value(self, user, field):
+        normalized = normalize_field(field)
+        name = normalized["name"]
+        callable_fn = normalized.get("callable")
+        subfields = normalized.get("fields")
+        # Priority: callable > fields > name
+        if callable_fn is not None:
+            try:
+                return callable_fn(user)
+            except Exception as e:
+                raise Exception(f"Error calling function for field '{name}': {e}")
+        if subfields is not None:
+            attr = resolve_attr(user, name)
+            if attr is None:
+                return ""
+            if hasattr(attr, "iterator"):
+                return serialize_related(attr, subfields)
+            return ",".join(str(getattr(attr, f, "")) for f in subfields)
+
+        # Dot-notation: e.g. "auth_token.key" or "profile.phone_number"
+        if "." in name:
+            model_attr, sub_attr = name.split(".", 1)
+            try:
+                intermediate = getattr(user, model_attr)
+            except ObjectDoesNotExist:
+                return ""
+            if hasattr(intermediate, "iterator"):
+                # Related manager accessed via dot notation → comma-separated values
+                return ",".join(
+                    str(getattr(obj, sub_attr, "")) for obj in intermediate.iterator()
+                )
+            try:
+                return getattr(intermediate, sub_attr)
+            except ObjectDoesNotExist:
+                return ""
+        return getattr(user, name)

openwisp_users/settings.py

@@ -13,6 +13,15 @@
 AUTH_BACKEND_AUTO_PREFIXES = getattr(
     settings, "OPENWISP_USERS_AUTH_BACKEND_AUTO_PREFIXES", tuple()
 )
+
+
+def _export_organizations(user):
+    return ",".join(
+        f'({org_id},{perm["is_admin"]})'
+        for org_id, perm in user.organizations_dict.items()
+    )
+
+
 EXPORT_USERS_COMMAND_CONFIG = {
     "fields": [
         "id",
@@ -29,9 +38,10 @@
         "location",
         "notes",
         "language",
-        "organizations",
+        {"name": "organizations", "callable": _export_organizations},
     ],
     "select_related": [],
+    "prefetch_related": [],
 }
 USER_PASSWORD_EXPIRATION = getattr(
     settings, "OPENWISP_USERS_USER_PASSWORD_EXPIRATION", 0